Merge pull request #3116 from pixelfed/staging

Store remote avatars locally if S3 not enabled

CHANGELOG.md

@@ -18,6 +18,7 @@
 - Added StatusMentionService, fixes #3026. ([e5387d67](https://github.com/pixelfed/pixelfed/commit/e5387d67))
 - Cloud Backups, a command to store backups on S3 or compatible filesystems. [#3037](https://github.com/pixelfed/pixelfed/pull/3037) ([3515a98e](https://github.com/pixelfed/pixelfed/commit/3515a98e))
 - Web UI Localizations + Crowdin integration. ([f7d9b40b](https://github.com/pixelfed/pixelfed/commit/f7d9b40b)) ([7ff120c9](https://github.com/pixelfed/pixelfed/commit/7ff120c9))
+- Store remote avatars locally if S3 not enabled. ([b4bd0400](https://github.com/pixelfed/pixelfed/commit/b4bd0400))
 
 ### Updated
 - Updated NotificationService, fix 500 bug. ([4a609dc3](https://github.com/pixelfed/pixelfed/commit/4a609dc3))

app/Console/Commands/AvatarSync.php

@@ -0,0 +1,219 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Avatar;
+use App\Profile;
+use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
+use App\Util\ActivityPub\Helpers;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
+
+class AvatarSync extends Command
+{
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'avatars:sync';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Perform actions on avatars';
+
+	public $found = 0;
+	public $notFetched = 0;
+	public $fixed = 0;
+
+	/**
+	 * Create a new command instance.
+	 *
+	 * @return void
+	 */
+	public function __construct()
+	{
+	    parent::__construct();
+	}
+
+	/**
+	 * Execute the console command.
+	 *
+	 * @return int
+	 */
+	public function handle()
+	{
+		$this->info('Welcome to the avatar sync manager');
+
+		$actions = [
+			'Analyze',
+			'Full Analyze',
+			'Fetch - Fetch missing remote avatars',
+			'Fix - Fix remote accounts without avatar record',
+			'Sync - Store latest remote avatars',
+		];
+
+		$name = $this->choice(
+			'Select an action',
+			$actions,
+			0,
+			1,
+			false
+		);
+
+		$this->info('Selected: ' . $name);
+
+		switch($name) {
+			case $actions[0]:
+				$this->analyze();
+			break;
+
+			case $actions[1]:
+				$this->fullAnalyze();
+			break;
+
+			case $actions[2]:
+				$this->fetch();
+			break;
+
+			case $actions[3]:
+				$this->fix();
+			break;
+
+			case $actions[4]:
+				$this->sync();
+			break;
+		}
+	    return Command::SUCCESS;
+	}
+
+	protected function incr($name)
+	{
+		switch($name) {
+			case 'found':
+				$this->found = $this->found + 1;
+			break;
+
+			case 'notFetched':
+				$this->notFetched = $this->notFetched + 1;
+			break;
+
+			case 'fixed':
+				$this->fixed++;
+			break;
+		}
+	}
+
+	protected function analyze()
+	{
+		$count = Avatar::whereIsRemote(true)->whereNull('cdn_url')->count();
+		$this->info('Found ' . $count . ' profiles with blank avatars.');
+		$this->line(' ');
+		$this->comment('We suggest running php artisan avatars:sync again and selecting the sync option');
+		$this->line(' ');
+	}
+
+	protected function fullAnalyze()
+	{
+		$count = Profile::count();
+		$bar = $this->output->createProgressBar($count);
+		$bar->start();
+
+		Profile::chunk(5000, function($profiles) use ($bar) {
+			foreach($profiles as $profile) {
+				if($profile->domain == null) {
+					$bar->advance();
+					continue;
+				}
+				$avatar = Avatar::whereProfileId($profile->id)->first();
+				if(!$avatar || $avatar->cdn_url == null) {
+					$this->incr('notFetched');
+				}
+				$this->incr('found');
+				$bar->advance();
+			}
+		});
+
+		$this->line(' ');
+		$this->line(' ');
+		$this->info('Found ' . $this->found . ' remote accounts');
+		$this->info('Found ' . $this->notFetched . ' remote avatars to fetch');
+	}
+
+	protected function fetch()
+	{
+		$this->info('Fetching ....');
+		Avatar::whereIsRemote(true)
+			->whereNull('cdn_url')
+			// ->with('profile')
+			->chunk(10, function($avatars) {
+				foreach($avatars as $avatar) {
+					if(!$avatar || !$avatar->profile) {
+						continue;
+					}
+					$url = $avatar->profile->remote_url;
+					if(!$url || !Helpers::validateUrl($url)) {
+						continue;
+					}
+					try {
+						$res = Helpers::fetchFromUrl($url);
+						if(
+							!is_array($res) ||
+							!isset($res['@context']) ||
+							!isset($res['icon']) ||
+							!isset($res['icon']['type']) ||
+							!isset($res['icon']['url']) ||
+							!Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg'])
+						) {
+							continue;
+						}
+					} catch (\GuzzleHttp\Exception\RequestException $e) {
+						continue;
+					} catch(\Illuminate\Http\Client\ConnectionException $e) {
+						continue;
+					}
+					$avatar->remote_url = $res['icon']['url'];
+					$avatar->save();
+					RemoteAvatarFetch::dispatch($avatar->profile);
+				}
+		});
+	}
+
+	protected function fix()
+	{
+		Profile::chunk(5000, function($profiles) {
+			foreach($profiles as $profile) {
+				if($profile->domain == null || $profile->private_key) {
+					continue;
+				}
+				$avatar = Avatar::whereProfileId($profile->id)->first();
+				if($avatar) {
+					continue;
+				}
+				$avatar = new Avatar;
+				$avatar->is_remote = true;
+				$avatar->profile_id = $profile->id;
+				$avatar->save();
+				$this->incr('fixed');
+			}
+		});
+		$this->line(' ');
+		$this->line(' ');
+		$this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar');
+	}
+
+	protected function sync()
+	{
+		Avatar::whereIsRemote(true)
+			->with('profile')
+			->chunk(10, function($avatars) {
+				foreach($avatars as $avatar) {
+					RemoteAvatarFetch::dispatch($avatar->profile);
+				}
+		});
+	}
+	}

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -32,6 +32,13 @@ class RemoteAvatarFetch implements ShouldQueue
 	*/
 	public $deleteWhenMissingModels = true;
 
+	/**
+	 * The number of times the job may be attempted.
+	 *
+	 * @var int
+	 */
+	public $tries = 1;
+
 	/**
 	* Create a new job instance.
 	*
@@ -99,9 +106,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		if(config_cache('pixelfed.cloud_storage')) {
-			MediaStorageService::avatar($avatar);
-		}
+		MediaStorageService::avatar($avatar, config_cache('pixelfed.cloud_storage') == false);
 
 		return 1;
 	}

app/Services/MediaStorageService.php

@@ -27,9 +27,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar)
+	public static function avatar($avatar, $local = false)
 	{
-		return (new self())->fetchAvatar($avatar);
+		return (new self())->fetchAvatar($avatar, $local);
 	}
 
 	public static function head($url)
@@ -177,11 +177,12 @@ class MediaStorageService {
 		unlink($tmpName);
 	}
 
-	protected function fetchAvatar($avatar)
+	protected function fetchAvatar($avatar, $local = false)
 	{
 		$url = $avatar->remote_url;
+		$driver = $local ? 'local' : config('filesystems.cloud');
 
-		if($url == null || Helpers::validateUrl($url) == false) {
+		if(empty($url) || Helpers::validateUrl($url) == false) {
 			return;
 		}
 
@@ -220,7 +221,7 @@ class MediaStorageService {
 			return;
 		}
 
-		$base = 'cache/avatars/' . $avatar->profile_id;
+		$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
 		$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
 		$path = Str::random(20) . '_avatar.' . $ext;
 		$tmpBase = storage_path('app/remcache/');
@@ -229,7 +230,7 @@ class MediaStorageService {
 		$data = file_get_contents($url, false, null, 0, $head['length']);
 		file_put_contents($tmpName, $data);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
+		$disk = Storage::disk($driver);
 		$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
 		$permalink = $disk->url($file);
 

app/Util/ActivityPub/Helpers.php

@@ -586,9 +586,7 @@ class Helpers {
 						$profile->webfinger = Purify::clean($webfinger);
 						$profile->last_fetched_at = now();
 						$profile->save();
-						if(config_cache('pixelfed.cloud_storage') == true) {
-							RemoteAvatarFetch::dispatch($profile);
-						}
+						RemoteAvatarFetch::dispatch($profile);
 						return $profile;
 					});
 				});
@@ -603,9 +601,7 @@ class Helpers {
 					$profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) && Helpers::validateUrl($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null;
 					$profile->save();
 				}
-				if(config_cache('pixelfed.cloud_storage') == true) {
-					RemoteAvatarFetch::dispatch($profile);
-				}
+				RemoteAvatarFetch::dispatch($profile);
 			}
 			return $profile;
 		});

config/federation.php

@@ -33,7 +33,7 @@ return [
 	],
 
 	'avatars' => [
-		'store_local' => false
+		'store_local' => env('REMOTE_AVATARS', true),
 	],
 
 	'nodeinfo' => [