UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-22 06:21:27 +00:00
Code Issues Projects Releases Packages Wiki Activity

Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars

Browse source
This commit is contained in:
Daniel Supernault 2023-10-11 00:48:30 -06:00
parent 36b23fe34e
commit 82798b5ea3
No known key found for this signature in database
GPG key ID: 23740873EE6F76A1
6 changed files with 270 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
app/Jobs/AvatarPipeline/AvatarStorageCleanup.php Normal file
View file

@ -0,0 +1,67 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
AvatarService::cleanup($this->avatar, true);
return;
}
}

80
app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php Normal file
View file

@ -0,0 +1,80 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
use Illuminate\Support\Str;
class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
$avatar = $this->avatar;
$disk = AvatarService::disk();
$files = collect(AvatarService::storage($avatar));
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

2
app/Jobs/AvatarPipeline/RemoteAvatarFetch.php
View file

@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
$avatar->remote_url = $icon['url'];
$avatar->save();
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;
}

1
app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php
View file

@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
$avatar->save();
}
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;

128
app/Services/AvatarService.php
View file

@ -3,21 +3,125 @@
namespace App\Services;
use Cache;
use Storage;
use Illuminate\Support\Str;
use App\Avatar;
use App\Profile;
use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
use League\Flysystem\UnableToCheckDirectoryExistence;
use League\Flysystem\UnableToRetrieveMetadata;
class AvatarService
{
public static function get($profile_id)
{
$exists = Cache::get('avatar:' . $profile_id);
if($exists) {
return $exists;
}
public static function get($profile_id)
{
$exists = Cache::get('avatar:' . $profile_id);
if($exists) {
return $exists;
}
$profile = Profile::find($profile_id);
if(!$profile) {
return config('app.url') . '/storage/avatars/default.jpg';
}
return $profile->avatarUrl();
}
$profile = Profile::find($profile_id);
if(!$profile) {
return config('app.url') . '/storage/avatars/default.jpg';
}
return $profile->avatarUrl();
}
public static function disk()
{
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return false;
}
$driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
$disk = Storage::disk($driver);
return $disk;
}
public static function storage(Avatar $avatar)
{
$disk = self::disk();
if(!$disk) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
return $disk->allFiles($base . $avatar->profile_id);
}
public static function cleanup($avatar, $confirm = false)
{
if(!$avatar || !$confirm) {
return;
}
if($avatar->cdn_url == null) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return;
}
$disk = self::disk();
if(!$disk) {
return;
}
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
try {
$exists = $disk->directoryExists($base . $avatar->profile_id);
} catch (
UnableToRetrieveMetadata |
UnableToCheckDirectoryExistence |
Exception $e
) {
return;
}
if(!$exists) {
return;
}
$files = collect($disk->allFiles($base . $avatar->profile_id));
if(!$files || !$files->count() || $files->count() === 1) {
return;
}
if($files->count() > 5) {
AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
return;
}
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

9
app/Services/MediaStorageService.php
View file

@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
use GuzzleHttp\Exception\RequestException;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
use Illuminate\Support\Arr;
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
class MediaStorageService {
@ -29,9 +30,9 @@ class MediaStorageService {
return;
}
public static function avatar($avatar, $local = false)
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
{
return (new self())->fetchAvatar($avatar, $local);
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
}
public static function head($url)
@ -182,6 +183,7 @@ class MediaStorageService {
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
{
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
$url = $avatar->remote_url;
$driver = $local ? 'local' : config('filesystems.cloud');
@ -205,7 +207,7 @@ class MediaStorageService {
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
if(!$skipRecentCheck) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
return;
}
}
@ -261,6 +263,7 @@ class MediaStorageService {
Cache::forget('avatar:' . $avatar->profile_id);
AccountService::del($avatar->profile_id);
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
unlink($tmpName);
}