Merge pull request #3875 from pixelfed/staging

Staging

CHANGELOG.md

@@ -30,6 +30,9 @@
 - Update InboxPipeline, bump request timeout from 5s to 60s ([bb120019](https://github.com/pixelfed/pixelfed/commit/bb120019))
 - Update web routes, fix missing home route ([a9f4ddfc](https://github.com/pixelfed/pixelfed/commit/a9f4ddfc))
 - Allow forceHttps to be disabled, fixes #3710 ([a31bdec7](https://github.com/pixelfed/pixelfed/commit/a31bdec7))
+- Update MediaStorageService, fix size check bug ([319f0ba5](https://github.com/pixelfed/pixelfed/commit/319f0ba5))
+- Update AvatarSync, fix sync skipping recently fetched avatars by setting last_fetched_at to null before refetching ([a83fc798](https://github.com/pixelfed/pixelfed/commit/a83fc798))
+- Refactor AvatarStorage to support migrating avatars to cloud storage, fix remote avatar refetching and merge AvatarSync commands and add deprecation notice to avatar:sync command ([223aea47](https://github.com/pixelfed/pixelfed/commit/223aea47))
 - ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

app/Avatar.php

@@ -20,7 +20,7 @@ class Avatar extends Model
         'last_processed_at'
     ];
     
-    protected $fillable = ['profile_id'];
+    protected $guarded = [];
 
     protected $visible = [
         'id',

app/Console/Commands/AvatarStorage.php

@@ -4,9 +4,14 @@ namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
 use App\Avatar;
+use App\Profile;
 use App\User;
+use Cache;
 use Storage;
+use App\Services\AccountService;
 use App\Util\Lexer\PrettyNumber;
+use Illuminate\Support\Str;
+use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
 
 class AvatarStorage extends Command
 {
@@ -24,6 +29,11 @@ class AvatarStorage extends Command
      */
     protected $description = 'Manage avatar storage';
 
+    public $found = 0;
+    public $notFetched = 0;
+    public $fixed = 0;
+    public $missing = 0;
+
     /**
      * Execute the console command.
      *
@@ -90,13 +100,25 @@ class AvatarStorage extends Command
             $this->info($msg);
         }
 
-        $choice = $this->choice(
+        $options = config_cache('pixelfed.cloud_storage') && config_cache('instance.avatar.local_to_cloud') ?
-            'Select action:',
             [
+                'Cancel',
                 'Upload default avatar to cloud',
                 'Move local avatars to cloud',
-                'Move cloud avatars to local'
+                'Re-fetch remote avatars'
-            ],
+            ] : [
+                'Cancel',
+                'Re-fetch remote avatars'
+        ];
+
+        $this->missing = Profile::where('created_at', '<', now()->subDays(1))->doesntHave('avatar')->count();
+        if($this->missing != 0) {
+            $options[] = 'Fix missing avatars';
+        }
+
+        $choice = $this->choice(
+            'Select action:',
+            $options,
             0
         );
 
@@ -106,18 +128,166 @@ class AvatarStorage extends Command
     protected function handleChoice($id)
     {
         switch ($id) {
+            case 'Cancel':
+                return;
+            break;
+
             case 'Upload default avatar to cloud':
                 return $this->uploadDefaultAvatar();
                 break;
+
+            case 'Move local avatars to cloud':
+                return $this->uploadAvatarsToCloud();
+                break;
+
+            case 'Re-fetch remote avatars':
+                return $this->refetchRemoteAvatars();
+                break;
+
+            case 'Fix missing avatars':
+                return $this->fixMissingAvatars();
+                break;
         }
     }
 
     protected function uploadDefaultAvatar()
     {
+        if(!$this->confirm('Are you sure you want to upload the default avatar to the cloud storage disk?')) {
+            return;
+        }
         $disk = Storage::disk(config_cache('filesystems.cloud'));
         $disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg'));
         Avatar::whereMediaPath('public/avatars/default.jpg')->update(['cdn_url' => $disk->url('cache/avatars/default.jpg')]);
         $this->info('Successfully uploaded default avatar to cloud storage!');
         $this->info($disk->url('cache/avatars/default.jpg'));
     }
+
+    protected function uploadAvatarsToCloud()
+    {
+        if(!config_cache('pixelfed.cloud_storage') || !config_cache('instance.avatar.local_to_cloud')) {
+            $this->error('Enable cloud storage and avatar cloud storage to perform this action');
+            return;
+        }
+        $confirm = $this->confirm('Are you sure you want to move local avatars to cloud storage?');
+        if(!$confirm) {
+            $this->error('Aborted action');
+            return;
+        }
+
+        $disk = Storage::disk(config_cache('filesystems.cloud'));
+
+        if($disk->missing('cache/avatars/default.jpg')) {
+            $disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg'));
+        }
+
+        Avatar::whereNull('is_remote')->chunk(5, function($avatars) use($disk) {
+            foreach($avatars as $avatar) {
+                if($avatar->media_path === 'public/avatars/default.jpg') {
+                    $avatar->cdn_url = $disk->url('cache/avatars/default.jpg');
+                    $avatar->save();
+                } else {
+                    if(!$avatar->media_path || !Str::of($avatar->media_path)->startsWith('public/avatars/')) {
+                        continue;
+                    }
+                    $ext = pathinfo($avatar->media_path, PATHINFO_EXTENSION);
+                    $newPath = 'cache/avatars/' . $avatar->profile_id . '/avatar_' . strtolower(Str::random(6)) . '.' . $ext;
+                    $existing = Storage::disk('local')->get($avatar->media_path);
+                    if(!$existing) {
+                        continue;
+                    }
+                    $newMediaPath = $disk->put($newPath, $existing);
+                    $avatar->media_path = $newMediaPath;
+                    $avatar->cdn_url = $disk->url($newMediaPath);
+                    $avatar->save();
+                }
+
+                Cache::forget('avatar:' . $avatar->profile_id);
+                Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id);
+            }
+        });
+    }
+
+    protected function refetchRemoteAvatars()
+    {
+        if(!$this->confirm('Are you sure you want to refetch all remote avatars? This could take a while.')) {
+            return;
+        }
+
+        if(config_cache('pixelfed.cloud_storage') == false && config_cache('federation.avatars.store_local') == false) {
+            $this->error('You have cloud storage disabled and local avatar storage disabled, we cannot refetch avatars.');
+            return;
+        }
+
+        $count = Profile::has('avatar')
+            ->with('avatar')
+            ->whereNull('user_id')
+            ->count();
+
+        $this->info('Found ' . $count . ' remote avatars to re-fetch');
+        $this->line(' ');
+        $bar = $this->output->createProgressBar($count);
+
+        Profile::has('avatar')
+            ->with('avatar')
+            ->whereNull('user_id')
+            ->chunk(50, function($profiles) use($bar) {
+            foreach($profiles as $profile) {
+                $avatar = $profile->avatar;
+                $avatar->last_fetched_at = null;
+                $avatar->save();
+                RemoteAvatarFetch::dispatch($profile)->onQueue('low');
+                $bar->advance();
+            }
+        });
+        $this->line(' ');
+        $this->line(' ');
+        $this->info('Finished dispatching avatar refetch jobs!');
+        $this->line(' ');
+        $this->info('This may take a few minutes to complete, you may need to run "php artisan cache:clear" after the jobs are processed.');
+        $this->line(' ');
+    }
+
+    protected function incr($name)
+    {
+        switch($name) {
+            case 'found':
+                $this->found = $this->found + 1;
+            break;
+
+            case 'notFetched':
+                $this->notFetched = $this->notFetched + 1;
+            break;
+
+            case 'fixed':
+                $this->fixed++;
+            break;
+        }
+    }
+
+    protected function fixMissingAvatars()
+    {
+        if(!$this->confirm('Are you sure you want to fix missing avatars?')) {
+            return;
+        }
+
+        $this->info('Found ' . $this->missing . ' accounts with missing profiles');
+
+        Profile::where('created_at', '<', now()->subDays(1))
+            ->doesntHave('avatar')
+            ->chunk(50, function($profiles) {
+                foreach($profiles as $profile) {
+                    Avatar::updateOrCreate([
+                        'profile_id' => $profile->id
+                    ], [
+                        'media_path' => 'public/avatars/default.jpg',
+                        'is_remote' => $profile->domain == null && $profile->private_key == null
+                    ]);
+                    $this->incr('fixed');
+                }
+        });
+
+        $this->line(' ');
+        $this->line(' ');
+        $this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar');
+    }
 }

app/Console/Commands/AvatarSync.php

@@ -48,6 +48,18 @@ class AvatarSync extends Command
 	public function handle()
 	{
 		$this->info('Welcome to the avatar sync manager');
+		$this->line(' ');
+		$this->line(' ');
+		$this->error('This command is deprecated and will be removed in a future version');
+		$this->error('You should use the following command instead: ');
+		$this->line(' ');
+		$this->info('php artisan avatar:storage');
+		$this->line(' ');
+
+		$confirm = $this->confirm('Are you sure you want to use this deprecated command even though it is no longer supported?');
+		if(!$confirm) {
+			return;
+		}
 
 		$actions = [
 			'Analyze',
@@ -123,7 +135,7 @@ class AvatarSync extends Command
 		$bar = $this->output->createProgressBar($count);
 		$bar->start();
 
-		Profile::chunk(5000, function($profiles) use ($bar) {
+		Profile::chunk(50, function($profiles) use ($bar) {
 			foreach($profiles as $profile) {
 				if($profile->domain == null) {
 					$bar->advance();
@@ -146,41 +158,11 @@ class AvatarSync extends Command
 
 	protected function fetch()
 	{
-		$this->info('Fetching ....');
+		$this->error('This action has been deprecated, please run the following command instead:');
-		Avatar::whereIsRemote(true)
+		$this->line(' ');
-			->whereNull('cdn_url')
+		$this->info('php artisan avatar:storage');
-			// ->with('profile')
+		$this->line(' ');
-			->chunk(10, function($avatars) {
+		return;
-				foreach($avatars as $avatar) {
-					if(!$avatar || !$avatar->profile) {
-						continue;
-					}
-					$url = $avatar->profile->remote_url;
-					if(!$url || !Helpers::validateUrl($url)) {
-						continue;
-					}
-					try {
-						$res = Helpers::fetchFromUrl($url);
-						if(
-							!is_array($res) ||
-							!isset($res['@context']) ||
-							!isset($res['icon']) ||
-							!isset($res['icon']['type']) ||
-							!isset($res['icon']['url']) ||
-							!Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg'])
-						) {
-							continue;
-						}
-					} catch (\GuzzleHttp\Exception\RequestException $e) {
-						continue;
-					} catch(\Illuminate\Http\Client\ConnectionException $e) {
-						continue;
-					}
-					$avatar->remote_url = $res['icon']['url'];
-					$avatar->save();
-					RemoteAvatarFetch::dispatch($avatar->profile);
-				}
-		});
 	}
 
 	protected function fix()
@@ -208,12 +190,10 @@ class AvatarSync extends Command
 
 	protected function sync()
 	{
-		Avatar::whereIsRemote(true)
+		$this->error('This action has been deprecated, please run the following command instead:');
-			->with('profile')
+		$this->line(' ');
-			->chunk(10, function($avatars) {
+		$this->info('php artisan avatar:storage');
-				foreach($avatars as $avatar) {
+		$this->line(' ');
-					RemoteAvatarFetch::dispatch($avatar->profile);
+		return;
-				}
-		});
-	}
 	}
+}

app/Services/MediaStorageService.php

@@ -12,6 +12,7 @@ use App\Media;
 use App\Profile;
 use App\User;
 use GuzzleHttp\Client;
+use App\Services\AccountService;
 use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
@@ -226,10 +227,6 @@ class MediaStorageService {
 			return;
 		}
 
-		if($avatar->size && $head['length'] == $avatar->size) {
-			return;
-		}
-
 		$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
 		$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
 		$path = Str::random(20) . '_avatar.' . $ext;
@@ -255,6 +252,7 @@ class MediaStorageService {
 		$avatar->save();
 
 		Cache::forget('avatar:' . $avatar->profile_id);
+		Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id);
 
 		unlink($tmpName);
 	}