mirror of
https://github.com/pixelfed/pixelfed.git
synced 2024-11-09 16:24:51 +00:00
Merge pull request #3116 from pixelfed/staging
Store remote avatars locally if S3 not enabled
This commit is contained in:
commit
a3c4d29b44
6 changed files with 238 additions and 16 deletions
|
@ -18,6 +18,7 @@
|
|||
- Added StatusMentionService, fixes #3026. ([e5387d67](https://github.com/pixelfed/pixelfed/commit/e5387d67))
|
||||
- Cloud Backups, a command to store backups on S3 or compatible filesystems. [#3037](https://github.com/pixelfed/pixelfed/pull/3037) ([3515a98e](https://github.com/pixelfed/pixelfed/commit/3515a98e))
|
||||
- Web UI Localizations + Crowdin integration. ([f7d9b40b](https://github.com/pixelfed/pixelfed/commit/f7d9b40b)) ([7ff120c9](https://github.com/pixelfed/pixelfed/commit/7ff120c9))
|
||||
- Store remote avatars locally if S3 not enabled. ([b4bd0400](https://github.com/pixelfed/pixelfed/commit/b4bd0400))
|
||||
|
||||
### Updated
|
||||
- Updated NotificationService, fix 500 bug. ([4a609dc3](https://github.com/pixelfed/pixelfed/commit/4a609dc3))
|
||||
|
|
219
app/Console/Commands/AvatarSync.php
Normal file
219
app/Console/Commands/AvatarSync.php
Normal file
|
@ -0,0 +1,219 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use App\Avatar;
|
||||
use App\Profile;
|
||||
use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
|
||||
use App\Util\ActivityPub\Helpers;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class AvatarSync extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'avatars:sync';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Perform actions on avatars';
|
||||
|
||||
public $found = 0;
|
||||
public $notFetched = 0;
|
||||
public $fixed = 0;
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->info('Welcome to the avatar sync manager');
|
||||
|
||||
$actions = [
|
||||
'Analyze',
|
||||
'Full Analyze',
|
||||
'Fetch - Fetch missing remote avatars',
|
||||
'Fix - Fix remote accounts without avatar record',
|
||||
'Sync - Store latest remote avatars',
|
||||
];
|
||||
|
||||
$name = $this->choice(
|
||||
'Select an action',
|
||||
$actions,
|
||||
0,
|
||||
1,
|
||||
false
|
||||
);
|
||||
|
||||
$this->info('Selected: ' . $name);
|
||||
|
||||
switch($name) {
|
||||
case $actions[0]:
|
||||
$this->analyze();
|
||||
break;
|
||||
|
||||
case $actions[1]:
|
||||
$this->fullAnalyze();
|
||||
break;
|
||||
|
||||
case $actions[2]:
|
||||
$this->fetch();
|
||||
break;
|
||||
|
||||
case $actions[3]:
|
||||
$this->fix();
|
||||
break;
|
||||
|
||||
case $actions[4]:
|
||||
$this->sync();
|
||||
break;
|
||||
}
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
protected function incr($name)
|
||||
{
|
||||
switch($name) {
|
||||
case 'found':
|
||||
$this->found = $this->found + 1;
|
||||
break;
|
||||
|
||||
case 'notFetched':
|
||||
$this->notFetched = $this->notFetched + 1;
|
||||
break;
|
||||
|
||||
case 'fixed':
|
||||
$this->fixed++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
protected function analyze()
|
||||
{
|
||||
$count = Avatar::whereIsRemote(true)->whereNull('cdn_url')->count();
|
||||
$this->info('Found ' . $count . ' profiles with blank avatars.');
|
||||
$this->line(' ');
|
||||
$this->comment('We suggest running php artisan avatars:sync again and selecting the sync option');
|
||||
$this->line(' ');
|
||||
}
|
||||
|
||||
protected function fullAnalyze()
|
||||
{
|
||||
$count = Profile::count();
|
||||
$bar = $this->output->createProgressBar($count);
|
||||
$bar->start();
|
||||
|
||||
Profile::chunk(5000, function($profiles) use ($bar) {
|
||||
foreach($profiles as $profile) {
|
||||
if($profile->domain == null) {
|
||||
$bar->advance();
|
||||
continue;
|
||||
}
|
||||
$avatar = Avatar::whereProfileId($profile->id)->first();
|
||||
if(!$avatar || $avatar->cdn_url == null) {
|
||||
$this->incr('notFetched');
|
||||
}
|
||||
$this->incr('found');
|
||||
$bar->advance();
|
||||
}
|
||||
});
|
||||
|
||||
$this->line(' ');
|
||||
$this->line(' ');
|
||||
$this->info('Found ' . $this->found . ' remote accounts');
|
||||
$this->info('Found ' . $this->notFetched . ' remote avatars to fetch');
|
||||
}
|
||||
|
||||
protected function fetch()
|
||||
{
|
||||
$this->info('Fetching ....');
|
||||
Avatar::whereIsRemote(true)
|
||||
->whereNull('cdn_url')
|
||||
// ->with('profile')
|
||||
->chunk(10, function($avatars) {
|
||||
foreach($avatars as $avatar) {
|
||||
if(!$avatar || !$avatar->profile) {
|
||||
continue;
|
||||
}
|
||||
$url = $avatar->profile->remote_url;
|
||||
if(!$url || !Helpers::validateUrl($url)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
$res = Helpers::fetchFromUrl($url);
|
||||
if(
|
||||
!is_array($res) ||
|
||||
!isset($res['@context']) ||
|
||||
!isset($res['icon']) ||
|
||||
!isset($res['icon']['type']) ||
|
||||
!isset($res['icon']['url']) ||
|
||||
!Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg'])
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
} catch (\GuzzleHttp\Exception\RequestException $e) {
|
||||
continue;
|
||||
} catch(\Illuminate\Http\Client\ConnectionException $e) {
|
||||
continue;
|
||||
}
|
||||
$avatar->remote_url = $res['icon']['url'];
|
||||
$avatar->save();
|
||||
RemoteAvatarFetch::dispatch($avatar->profile);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function fix()
|
||||
{
|
||||
Profile::chunk(5000, function($profiles) {
|
||||
foreach($profiles as $profile) {
|
||||
if($profile->domain == null || $profile->private_key) {
|
||||
continue;
|
||||
}
|
||||
$avatar = Avatar::whereProfileId($profile->id)->first();
|
||||
if($avatar) {
|
||||
continue;
|
||||
}
|
||||
$avatar = new Avatar;
|
||||
$avatar->is_remote = true;
|
||||
$avatar->profile_id = $profile->id;
|
||||
$avatar->save();
|
||||
$this->incr('fixed');
|
||||
}
|
||||
});
|
||||
$this->line(' ');
|
||||
$this->line(' ');
|
||||
$this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar');
|
||||
}
|
||||
|
||||
protected function sync()
|
||||
{
|
||||
Avatar::whereIsRemote(true)
|
||||
->with('profile')
|
||||
->chunk(10, function($avatars) {
|
||||
foreach($avatars as $avatar) {
|
||||
RemoteAvatarFetch::dispatch($avatar->profile);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
|
@ -32,6 +32,13 @@ class RemoteAvatarFetch implements ShouldQueue
|
|||
*/
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* The number of times the job may be attempted.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $tries = 1;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
|
@ -99,9 +106,7 @@ class RemoteAvatarFetch implements ShouldQueue
|
|||
$avatar->remote_url = $icon['url'];
|
||||
$avatar->save();
|
||||
|
||||
if(config_cache('pixelfed.cloud_storage')) {
|
||||
MediaStorageService::avatar($avatar);
|
||||
}
|
||||
MediaStorageService::avatar($avatar, config_cache('pixelfed.cloud_storage') == false);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
|
|
@ -27,9 +27,9 @@ class MediaStorageService {
|
|||
return;
|
||||
}
|
||||
|
||||
public static function avatar($avatar)
|
||||
public static function avatar($avatar, $local = false)
|
||||
{
|
||||
return (new self())->fetchAvatar($avatar);
|
||||
return (new self())->fetchAvatar($avatar, $local);
|
||||
}
|
||||
|
||||
public static function head($url)
|
||||
|
@ -177,11 +177,12 @@ class MediaStorageService {
|
|||
unlink($tmpName);
|
||||
}
|
||||
|
||||
protected function fetchAvatar($avatar)
|
||||
protected function fetchAvatar($avatar, $local = false)
|
||||
{
|
||||
$url = $avatar->remote_url;
|
||||
$driver = $local ? 'local' : config('filesystems.cloud');
|
||||
|
||||
if($url == null || Helpers::validateUrl($url) == false) {
|
||||
if(empty($url) || Helpers::validateUrl($url) == false) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -220,7 +221,7 @@ class MediaStorageService {
|
|||
return;
|
||||
}
|
||||
|
||||
$base = 'cache/avatars/' . $avatar->profile_id;
|
||||
$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
|
||||
$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
|
||||
$path = Str::random(20) . '_avatar.' . $ext;
|
||||
$tmpBase = storage_path('app/remcache/');
|
||||
|
@ -229,7 +230,7 @@ class MediaStorageService {
|
|||
$data = file_get_contents($url, false, null, 0, $head['length']);
|
||||
file_put_contents($tmpName, $data);
|
||||
|
||||
$disk = Storage::disk(config('filesystems.cloud'));
|
||||
$disk = Storage::disk($driver);
|
||||
$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
|
||||
$permalink = $disk->url($file);
|
||||
|
||||
|
|
|
@ -586,9 +586,7 @@ class Helpers {
|
|||
$profile->webfinger = Purify::clean($webfinger);
|
||||
$profile->last_fetched_at = now();
|
||||
$profile->save();
|
||||
if(config_cache('pixelfed.cloud_storage') == true) {
|
||||
RemoteAvatarFetch::dispatch($profile);
|
||||
}
|
||||
RemoteAvatarFetch::dispatch($profile);
|
||||
return $profile;
|
||||
});
|
||||
});
|
||||
|
@ -603,9 +601,7 @@ class Helpers {
|
|||
$profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) && Helpers::validateUrl($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null;
|
||||
$profile->save();
|
||||
}
|
||||
if(config_cache('pixelfed.cloud_storage') == true) {
|
||||
RemoteAvatarFetch::dispatch($profile);
|
||||
}
|
||||
RemoteAvatarFetch::dispatch($profile);
|
||||
}
|
||||
return $profile;
|
||||
});
|
||||
|
|
|
@ -33,7 +33,7 @@ return [
|
|||
],
|
||||
|
||||
'avatars' => [
|
||||
'store_local' => false
|
||||
'store_local' => env('REMOTE_AVATARS', true),
|
||||
],
|
||||
|
||||
'nodeinfo' => [
|
||||
|
|
Loading…
Reference in a new issue