Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars

app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+
+class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        AvatarService::cleanup($this->avatar, true);
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+use Illuminate\Support\Str;
+
+class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $avatar = $this->avatar;
+
+        $disk = AvatarService::disk();
+
+        $files = collect(AvatarService::storage($avatar));
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
+		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;
 	}

app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

@@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
 			$avatar->save();
 		}
 
-
 		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;

app/Services/AvatarService.php

@@ -3,21 +3,125 @@
 namespace App\Services;
 
 use Cache;
+use Storage;
+use Illuminate\Support\Str;
+use App\Avatar;
 use App\Profile;
+use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
+use League\Flysystem\UnableToCheckDirectoryExistence;
+use League\Flysystem\UnableToRetrieveMetadata;
 
 class AvatarService
 {
-	public static function get($profile_id)
+    public static function get($profile_id)
-	{
+    {
-		$exists = Cache::get('avatar:' . $profile_id);
+        $exists = Cache::get('avatar:' . $profile_id);
-		if($exists) {
+        if($exists) {
-			return $exists;
+            return $exists;
-		}
+        }
 
-		$profile = Profile::find($profile_id);
+        $profile = Profile::find($profile_id);
-		if(!$profile) {
+        if(!$profile) {
-			return config('app.url') . '/storage/avatars/default.jpg';
+            return config('app.url') . '/storage/avatars/default.jpg';
-		}
+        }
-		return $profile->avatarUrl();
+        return $profile->avatarUrl();
-	}
+    }
+
+    public static function disk()
+    {
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return false;
+        }
+
+        $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
+        $disk = Storage::disk($driver);
+
+        return $disk;
+    }
+
+    public static function storage(Avatar $avatar)
+    {
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        return $disk->allFiles($base . $avatar->profile_id);
+    }
+
+    public static function cleanup($avatar, $confirm = false)
+    {
+        if(!$avatar || !$confirm) {
+            return;
+        }
+
+        if($avatar->cdn_url == null) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return;
+        }
+
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        try {
+            $exists = $disk->directoryExists($base . $avatar->profile_id);
+        } catch (
+            UnableToRetrieveMetadata |
+            UnableToCheckDirectoryExistence |
+            Exception $e
+        ) {
+            return;
+        }
+
+        if(!$exists) {
+            return;
+        }
+
+        $files = collect($disk->allFiles($base . $avatar->profile_id));
+
+        if(!$files || !$files->count() || $files->count() === 1) {
+            return;
+        }
+
+        if($files->count() > 5) {
+            AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
+            return;
+        }
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
 }

app/Services/MediaStorageService.php

@@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 use Illuminate\Support\Arr;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
 
 class MediaStorageService {
 
@@ -29,9 +30,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar, $local = false)
+	public static function avatar($avatar, $local = false, $skipRecentCheck = false)
 	{
-		return (new self())->fetchAvatar($avatar, $local);
+		return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
 	}
 
 	public static function head($url)
@@ -182,6 +183,7 @@ class MediaStorageService {
 
 	protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
 	{
+		$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
 		$url = $avatar->remote_url;
 		$driver = $local ? 'local' : config('filesystems.cloud');
 
@@ -205,7 +207,7 @@ class MediaStorageService {
 		$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
 
 		if(!$skipRecentCheck) {
-			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
+			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
 				return;
 			}
 		}
@@ -261,6 +263,7 @@ class MediaStorageService {
 
 		Cache::forget('avatar:' . $avatar->profile_id);
 		AccountService::del($avatar->profile_id);
+		AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
 
 		unlink($tmpName);
 	}