From 82798b5ea3e77867cb06981ac412a3aee781752e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 00:48:30 -0600 Subject: [PATCH] Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars --- .../AvatarPipeline/AvatarStorageCleanup.php | 67 +++++++++ .../AvatarStorageLargePurge.php | 80 +++++++++++ app/Jobs/AvatarPipeline/RemoteAvatarFetch.php | 2 +- .../RemoteAvatarFetchFromUrl.php | 1 - app/Services/AvatarService.php | 128 ++++++++++++++++-- app/Services/MediaStorageService.php | 9 +- 6 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageCleanup.php create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php diff --git a/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php new file mode 100644 index 000000000..230797bf6 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php @@ -0,0 +1,67 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + AvatarService::cleanup($this->avatar, true); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php new file mode 100644 index 000000000..f432e1e56 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php @@ -0,0 +1,80 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $avatar = $this->avatar; + + $disk = AvatarService::disk(); + + $files = collect(AvatarService::storage($avatar)); + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php index df972dd38..4e4a1b2ec 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php @@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue $avatar->remote_url = $icon['url']; $avatar->save(); - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false); + MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; } diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php index 259058385..c8c6820e4 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php @@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue $avatar->save(); } - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; diff --git a/app/Services/AvatarService.php b/app/Services/AvatarService.php index 1c5e9e0c1..af578fdef 100644 --- a/app/Services/AvatarService.php +++ b/app/Services/AvatarService.php @@ -3,21 +3,125 @@ namespace App\Services; use Cache; +use Storage; +use Illuminate\Support\Str; +use App\Avatar; use App\Profile; +use App\Jobs\AvatarPipeline\AvatarStorageLargePurge; +use League\Flysystem\UnableToCheckDirectoryExistence; +use League\Flysystem\UnableToRetrieveMetadata; class AvatarService { - public static function get($profile_id) - { - $exists = Cache::get('avatar:' . $profile_id); - if($exists) { - return $exists; - } + public static function get($profile_id) + { + $exists = Cache::get('avatar:' . $profile_id); + if($exists) { + return $exists; + } - $profile = Profile::find($profile_id); - if(!$profile) { - return config('app.url') . '/storage/avatars/default.jpg'; - } - return $profile->avatarUrl(); - } + $profile = Profile::find($profile_id); + if(!$profile) { + return config('app.url') . '/storage/avatars/default.jpg'; + } + return $profile->avatarUrl(); + } + + public static function disk() + { + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return false; + } + + $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud'); + $disk = Storage::disk($driver); + + return $disk; + } + + public static function storage(Avatar $avatar) + { + $disk = self::disk(); + + if(!$disk) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + return $disk->allFiles($base . $avatar->profile_id); + } + + public static function cleanup($avatar, $confirm = false) + { + if(!$avatar || !$confirm) { + return; + } + + if($avatar->cdn_url == null) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return; + } + + $disk = self::disk(); + + if(!$disk) { + return; + } + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + try { + $exists = $disk->directoryExists($base . $avatar->profile_id); + } catch ( + UnableToRetrieveMetadata | + UnableToCheckDirectoryExistence | + Exception $e + ) { + return; + } + + if(!$exists) { + return; + } + + $files = collect($disk->allFiles($base . $avatar->profile_id)); + + if(!$files || !$files->count() || $files->count() === 1) { + return; + } + + if($files->count() > 5) { + AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo'); + return; + } + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } } diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b547ee39c..128001de2 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController; use GuzzleHttp\Exception\RequestException; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Illuminate\Support\Arr; +use App\Jobs\AvatarPipeline\AvatarStorageCleanup; class MediaStorageService { @@ -29,9 +30,9 @@ class MediaStorageService { return; } - public static function avatar($avatar, $local = false) + public static function avatar($avatar, $local = false, $skipRecentCheck = false) { - return (new self())->fetchAvatar($avatar, $local); + return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); } public static function head($url) @@ -182,6 +183,7 @@ class MediaStorageService { protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) { + $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; $url = $avatar->remote_url; $driver = $local ? 'local' : config('filesystems.cloud'); @@ -205,7 +207,7 @@ class MediaStorageService { $max_size = (int) config('pixelfed.max_avatar_size') * 1000; if(!$skipRecentCheck) { - if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) { + if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { return; } } @@ -261,6 +263,7 @@ class MediaStorageService { Cache::forget('avatar:' . $avatar->profile_id); AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); unlink($tmpName); }