From 319f0ba50f21cb23277776f389ac0eded5beac7a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 1 Dec 2022 23:13:44 -0700 Subject: [PATCH 1/4] Update MediaStorageService, fix size check bug --- app/Services/MediaStorageService.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index 1c3272ec1..8dd8319e7 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -12,6 +12,7 @@ use App\Media; use App\Profile; use App\User; use GuzzleHttp\Client; +use App\Services\AccountService; use App\Http\Controllers\AvatarController; use GuzzleHttp\Exception\RequestException; use App\Jobs\MediaPipeline\MediaDeletePipeline; @@ -226,10 +227,6 @@ class MediaStorageService { return; } - if($avatar->size && $head['length'] == $avatar->size) { - return; - } - $base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id; $ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png'; $path = Str::random(20) . '_avatar.' . $ext; @@ -255,6 +252,7 @@ class MediaStorageService { $avatar->save(); Cache::forget('avatar:' . $avatar->profile_id); + Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id); unlink($tmpName); } From a83fc798b7651a2e63b2a323bbe2a71401001b45 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 1 Dec 2022 23:17:05 -0700 Subject: [PATCH 2/4] Update AvatarSync, fix sync skipping recently fetched avatars by setting last_fetched_at to null before refetching --- app/Console/Commands/AvatarSync.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/AvatarSync.php b/app/Console/Commands/AvatarSync.php index 57d984747..e936b722c 100644 --- a/app/Console/Commands/AvatarSync.php +++ b/app/Console/Commands/AvatarSync.php @@ -212,7 +212,9 @@ class AvatarSync extends Command ->with('profile') ->chunk(10, function($avatars) { foreach($avatars as $avatar) { - RemoteAvatarFetch::dispatch($avatar->profile); + $avatar->last_fetched_at = null; + $avatar->save(); + RemoteAvatarFetch::dispatch($avatar->profile)->onQueue('low'); } }); } From 223aea476571e46b06899ab1bf9596436e7614ce Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 2 Dec 2022 00:21:53 -0700 Subject: [PATCH 3/4] Refactor AvatarStorage to support migrating avatars to cloud storage, fix remote avatar refetching and merge AvatarSync commands and add deprecation notice to avatar:sync command --- app/Avatar.php | 2 +- app/Console/Commands/AvatarStorage.php | 178 ++++++++++++++++++++++++- app/Console/Commands/AvatarSync.php | 70 ++++------ 3 files changed, 199 insertions(+), 51 deletions(-) diff --git a/app/Avatar.php b/app/Avatar.php index 0dde3841f..d8eece36a 100644 --- a/app/Avatar.php +++ b/app/Avatar.php @@ -20,7 +20,7 @@ class Avatar extends Model 'last_processed_at' ]; - protected $fillable = ['profile_id']; + protected $guarded = []; protected $visible = [ 'id', diff --git a/app/Console/Commands/AvatarStorage.php b/app/Console/Commands/AvatarStorage.php index b0d388053..1568d335c 100644 --- a/app/Console/Commands/AvatarStorage.php +++ b/app/Console/Commands/AvatarStorage.php @@ -4,9 +4,14 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\Avatar; +use App\Profile; use App\User; +use Cache; use Storage; +use App\Services\AccountService; use App\Util\Lexer\PrettyNumber; +use Illuminate\Support\Str; +use App\Jobs\AvatarPipeline\RemoteAvatarFetch; class AvatarStorage extends Command { @@ -24,6 +29,11 @@ class AvatarStorage extends Command */ protected $description = 'Manage avatar storage'; + public $found = 0; + public $notFetched = 0; + public $fixed = 0; + public $missing = 0; + /** * Execute the console command. * @@ -90,13 +100,25 @@ class AvatarStorage extends Command $this->info($msg); } - $choice = $this->choice( - 'Select action:', + $options = config_cache('pixelfed.cloud_storage') && config_cache('instance.avatar.local_to_cloud') ? [ + 'Cancel', 'Upload default avatar to cloud', 'Move local avatars to cloud', - 'Move cloud avatars to local' - ], + 'Re-fetch remote avatars' + ] : [ + 'Cancel', + 'Re-fetch remote avatars' + ]; + + $this->missing = Profile::where('created_at', '<', now()->subDays(1))->doesntHave('avatar')->count(); + if($this->missing != 0) { + $options[] = 'Fix missing avatars'; + } + + $choice = $this->choice( + 'Select action:', + $options, 0 ); @@ -106,18 +128,166 @@ class AvatarStorage extends Command protected function handleChoice($id) { switch ($id) { + case 'Cancel': + return; + break; + case 'Upload default avatar to cloud': return $this->uploadDefaultAvatar(); break; + + case 'Move local avatars to cloud': + return $this->uploadAvatarsToCloud(); + break; + + case 'Re-fetch remote avatars': + return $this->refetchRemoteAvatars(); + break; + + case 'Fix missing avatars': + return $this->fixMissingAvatars(); + break; } } protected function uploadDefaultAvatar() { + if(!$this->confirm('Are you sure you want to upload the default avatar to the cloud storage disk?')) { + return; + } $disk = Storage::disk(config_cache('filesystems.cloud')); $disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg')); Avatar::whereMediaPath('public/avatars/default.jpg')->update(['cdn_url' => $disk->url('cache/avatars/default.jpg')]); $this->info('Successfully uploaded default avatar to cloud storage!'); $this->info($disk->url('cache/avatars/default.jpg')); } + + protected function uploadAvatarsToCloud() + { + if(!config_cache('pixelfed.cloud_storage') || !config_cache('instance.avatar.local_to_cloud')) { + $this->error('Enable cloud storage and avatar cloud storage to perform this action'); + return; + } + $confirm = $this->confirm('Are you sure you want to move local avatars to cloud storage?'); + if(!$confirm) { + $this->error('Aborted action'); + return; + } + + $disk = Storage::disk(config_cache('filesystems.cloud')); + + if($disk->missing('cache/avatars/default.jpg')) { + $disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg')); + } + + Avatar::whereNull('is_remote')->chunk(5, function($avatars) use($disk) { + foreach($avatars as $avatar) { + if($avatar->media_path === 'public/avatars/default.jpg') { + $avatar->cdn_url = $disk->url('cache/avatars/default.jpg'); + $avatar->save(); + } else { + if(!$avatar->media_path || !Str::of($avatar->media_path)->startsWith('public/avatars/')) { + continue; + } + $ext = pathinfo($avatar->media_path, PATHINFO_EXTENSION); + $newPath = 'cache/avatars/' . $avatar->profile_id . '/avatar_' . strtolower(Str::random(6)) . '.' . $ext; + $existing = Storage::disk('local')->get($avatar->media_path); + if(!$existing) { + continue; + } + $newMediaPath = $disk->put($newPath, $existing); + $avatar->media_path = $newMediaPath; + $avatar->cdn_url = $disk->url($newMediaPath); + $avatar->save(); + } + + Cache::forget('avatar:' . $avatar->profile_id); + Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id); + } + }); + } + + protected function refetchRemoteAvatars() + { + if(!$this->confirm('Are you sure you want to refetch all remote avatars? This could take a while.')) { + return; + } + + if(config_cache('pixelfed.cloud_storage') == false && config_cache('federation.avatars.store_local') == false) { + $this->error('You have cloud storage disabled and local avatar storage disabled, we cannot refetch avatars.'); + return; + } + + $count = Profile::has('avatar') + ->with('avatar') + ->whereNull('user_id') + ->count(); + + $this->info('Found ' . $count . ' remote avatars to re-fetch'); + $this->line(' '); + $bar = $this->output->createProgressBar($count); + + Profile::has('avatar') + ->with('avatar') + ->whereNull('user_id') + ->chunk(50, function($profiles) use($bar) { + foreach($profiles as $profile) { + $avatar = $profile->avatar; + $avatar->last_fetched_at = null; + $avatar->save(); + RemoteAvatarFetch::dispatch($profile)->onQueue('low'); + $bar->advance(); + } + }); + $this->line(' '); + $this->line(' '); + $this->info('Finished dispatching avatar refetch jobs!'); + $this->line(' '); + $this->info('This may take a few minutes to complete, you may need to run "php artisan cache:clear" after the jobs are processed.'); + $this->line(' '); + } + + protected function incr($name) + { + switch($name) { + case 'found': + $this->found = $this->found + 1; + break; + + case 'notFetched': + $this->notFetched = $this->notFetched + 1; + break; + + case 'fixed': + $this->fixed++; + break; + } + } + + protected function fixMissingAvatars() + { + if(!$this->confirm('Are you sure you want to fix missing avatars?')) { + return; + } + + $this->info('Found ' . $this->missing . ' accounts with missing profiles'); + + Profile::where('created_at', '<', now()->subDays(1)) + ->doesntHave('avatar') + ->chunk(50, function($profiles) { + foreach($profiles as $profile) { + Avatar::updateOrCreate([ + 'profile_id' => $profile->id + ], [ + 'media_path' => 'public/avatars/default.jpg', + 'is_remote' => $profile->domain == null && $profile->private_key == null + ]); + $this->incr('fixed'); + } + }); + + $this->line(' '); + $this->line(' '); + $this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar'); + } } diff --git a/app/Console/Commands/AvatarSync.php b/app/Console/Commands/AvatarSync.php index e936b722c..aaede1c71 100644 --- a/app/Console/Commands/AvatarSync.php +++ b/app/Console/Commands/AvatarSync.php @@ -48,6 +48,18 @@ class AvatarSync extends Command public function handle() { $this->info('Welcome to the avatar sync manager'); + $this->line(' '); + $this->line(' '); + $this->error('This command is deprecated and will be removed in a future version'); + $this->error('You should use the following command instead: '); + $this->line(' '); + $this->info('php artisan avatar:storage'); + $this->line(' '); + + $confirm = $this->confirm('Are you sure you want to use this deprecated command even though it is no longer supported?'); + if(!$confirm) { + return; + } $actions = [ 'Analyze', @@ -123,7 +135,7 @@ class AvatarSync extends Command $bar = $this->output->createProgressBar($count); $bar->start(); - Profile::chunk(5000, function($profiles) use ($bar) { + Profile::chunk(50, function($profiles) use ($bar) { foreach($profiles as $profile) { if($profile->domain == null) { $bar->advance(); @@ -146,41 +158,11 @@ class AvatarSync extends Command protected function fetch() { - $this->info('Fetching ....'); - Avatar::whereIsRemote(true) - ->whereNull('cdn_url') - // ->with('profile') - ->chunk(10, function($avatars) { - foreach($avatars as $avatar) { - if(!$avatar || !$avatar->profile) { - continue; - } - $url = $avatar->profile->remote_url; - if(!$url || !Helpers::validateUrl($url)) { - continue; - } - try { - $res = Helpers::fetchFromUrl($url); - if( - !is_array($res) || - !isset($res['@context']) || - !isset($res['icon']) || - !isset($res['icon']['type']) || - !isset($res['icon']['url']) || - !Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg']) - ) { - continue; - } - } catch (\GuzzleHttp\Exception\RequestException $e) { - continue; - } catch(\Illuminate\Http\Client\ConnectionException $e) { - continue; - } - $avatar->remote_url = $res['icon']['url']; - $avatar->save(); - RemoteAvatarFetch::dispatch($avatar->profile); - } - }); + $this->error('This action has been deprecated, please run the following command instead:'); + $this->line(' '); + $this->info('php artisan avatar:storage'); + $this->line(' '); + return; } protected function fix() @@ -208,14 +190,10 @@ class AvatarSync extends Command protected function sync() { - Avatar::whereIsRemote(true) - ->with('profile') - ->chunk(10, function($avatars) { - foreach($avatars as $avatar) { - $avatar->last_fetched_at = null; - $avatar->save(); - RemoteAvatarFetch::dispatch($avatar->profile)->onQueue('low'); - } - }); - } + $this->error('This action has been deprecated, please run the following command instead:'); + $this->line(' '); + $this->info('php artisan avatar:storage'); + $this->line(' '); + return; } +} From 1f45fa9b39faeb8bbf193914c2dc4be356c56b0c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 2 Dec 2022 00:22:40 -0700 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 212f2abd5..80eff9951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ - Update InboxPipeline, bump request timeout from 5s to 60s ([bb120019](https://github.com/pixelfed/pixelfed/commit/bb120019)) - Update web routes, fix missing home route ([a9f4ddfc](https://github.com/pixelfed/pixelfed/commit/a9f4ddfc)) - Allow forceHttps to be disabled, fixes #3710 ([a31bdec7](https://github.com/pixelfed/pixelfed/commit/a31bdec7)) +- Update MediaStorageService, fix size check bug ([319f0ba5](https://github.com/pixelfed/pixelfed/commit/319f0ba5)) +- Update AvatarSync, fix sync skipping recently fetched avatars by setting last_fetched_at to null before refetching ([a83fc798](https://github.com/pixelfed/pixelfed/commit/a83fc798)) +- Refactor AvatarStorage to support migrating avatars to cloud storage, fix remote avatar refetching and merge AvatarSync commands and add deprecation notice to avatar:sync command ([223aea47](https://github.com/pixelfed/pixelfed/commit/223aea47)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)