UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-10 00:34:50 +00:00
Code Issues Projects Releases Packages Wiki Activity

Update MediaS3GarbageCollector commmand, disable logging by default and optimize huge invocations

Browse source
This commit is contained in:
Daniel Supernault 2022-12-17 21:44:01 -07:00
parent e503c83b6c
commit a14af93b02
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
1 changed files with 57 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
app/Console/Commands/MediaS3GarbageCollector.php
View file

@ -15,7 +15,7 @@ class MediaS3GarbageCollector extends Command
* *
* @var string * @var string
*/ */
protected $signature = 'media:s3gc {--limit=200}'; protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
/** /**
* The console command description. * The console command description.
@ -54,8 +54,23 @@ class MediaS3GarbageCollector extends Command
} }
$limit = $this->option('limit'); $limit = $this->option('limit');
$hugeMode = $this->option('huge');
$log = $this->option('log-errors');
if($limit > 2000 && !$hugeMode) {
$this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
return;
}
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id; $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id;
return $hugeMode ?
$this->hugeMode($minId, $limit, $log) :
$this->regularMode($minId, $limit, $log);
}
protected function regularMode($minId, $limit, $log)
{
$gc = Media::whereRemoteMedia(false) $gc = Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4') ->whereNot('version', '4')
@ -84,8 +99,10 @@ class MediaS3GarbageCollector extends Command
$media->save(); $media->save();
} }
} else { } else {
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
} }
}
$bar->advance(); $bar->advance();
} }
$bar->finish(); $bar->finish();
@ -96,4 +113,42 @@ class MediaS3GarbageCollector extends Command
} }
return 0; return 0;
} }
protected function hugeMode($minId, $limit, $log)
{
$cloudDisk = Storage::disk(config('filesystems.cloud'));
$localDisk = Storage::disk('local');
$bar = $this->output->createProgressBar($limit);
$bar->start();
Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
->where('id', '<', $minId)
->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
foreach($medias as $media) {
if($cloudDisk->exists($media->media_path)) {
if( $localDisk->exists($media->media_path)) {
$localDisk->delete($media->media_path);
$media->version = 4;
$media->save();
$totalSize = $totalSize + $media->size;
} else {
$media->version = 4;
$media->save();
}
} else {
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
}
$bar->advance();
}
});
$bar->finish();
$this->line(' ');
$this->info('Finished!');
}
} }