pixelfed/app/Console/Commands/MediaS3GarbageCollector.php

196 lines
6.4 KiB
PHP
Raw Permalink Normal View History

<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Media;
use App\Status;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
2022-12-18 05:25:57 +00:00
use App\Services\MediaService;
use App\Services\StatusService;
2022-12-18 07:40:59 +00:00
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
class MediaS3GarbageCollector extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Delete (local) media uploads that exist on S3';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
2024-03-12 07:03:33 +00:00
$enabled = (bool) config_cache('pixelfed.cloud_storage');
if(!$enabled) {
$this->error('Cloud storage not enabled. Exiting...');
return;
}
$deleteEnabled = config('media.delete_local_after_cloud');
if(!$deleteEnabled) {
$this->error('Delete local storage after cloud upload is not enabled');
return;
}
$limit = $this->option('limit');
$hugeMode = $this->option('huge');
$log = $this->option('log-errors');
if($limit > 2000 && !$hugeMode) {
$this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
return;
}
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first();
if(!$minId) {
return;
} else {
$minId = $minId->id;
}
return $hugeMode ?
$this->hugeMode($minId, $limit, $log) :
$this->regularMode($minId, $limit, $log);
}
protected function regularMode($minId, $limit, $log)
{
2022-12-18 03:47:47 +00:00
$gc = Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
->where('id', '<', $minId)
->inRandomOrder()
->take($limit)
->get();
$totalSize = 0;
$bar = $this->output->createProgressBar($gc->count());
$bar->start();
$cloudDisk = Storage::disk(config('filesystems.cloud'));
$localDisk = Storage::disk('local');
foreach($gc as $media) {
2022-12-18 07:40:59 +00:00
try {
if(
$cloudDisk->exists($media->media_path)
) {
if( $localDisk->exists($media->media_path)) {
$localDisk->delete($media->media_path);
$media->version = 4;
$media->save();
$totalSize = $totalSize + $media->size;
MediaService::del($media->status_id);
StatusService::del($media->status_id, false);
if($localDisk->exists($media->thumbnail_path)) {
$localDisk->delete($media->thumbnail_path);
}
2022-12-18 07:40:59 +00:00
} else {
$media->version = 4;
$media->save();
}
} else {
2022-12-18 07:40:59 +00:00
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
}
2022-12-18 07:40:59 +00:00
$bar->advance();
} catch (FileNotFoundException $e) {
$bar->advance();
continue;
} catch (NotFoundHttpException $e) {
$bar->advance();
continue;
} catch (\Exception $e) {
$bar->advance();
continue;
}
}
$bar->finish();
$this->line(' ');
$this->info('Finished!');
if($totalSize) {
$this->info('Cleared ' . $totalSize . ' bytes of media from local disk!');
}
return 0;
}
protected function hugeMode($minId, $limit, $log)
{
$cloudDisk = Storage::disk(config('filesystems.cloud'));
$localDisk = Storage::disk('local');
$bar = $this->output->createProgressBar($limit);
$bar->start();
Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
->where('id', '<', $minId)
->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
foreach($medias as $media) {
2022-12-18 07:40:59 +00:00
try {
if($cloudDisk->exists($media->media_path)) {
if( $localDisk->exists($media->media_path)) {
$localDisk->delete($media->media_path);
$media->version = 4;
$media->save();
MediaService::del($media->status_id);
StatusService::del($media->status_id, false);
if($localDisk->exists($media->thumbnail_path)) {
$localDisk->delete($media->thumbnail_path);
}
2022-12-18 07:40:59 +00:00
} else {
$media->version = 4;
$media->save();
}
} else {
2022-12-18 07:40:59 +00:00
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
}
2022-12-18 07:40:59 +00:00
$bar->advance();
} catch (FileNotFoundException $e) {
$bar->advance();
continue;
} catch (NotFoundHttpException $e) {
$bar->advance();
continue;
} catch (\Exception $e) {
$bar->advance();
continue;
}
}
});
$bar->finish();
$this->line(' ');
$this->info('Finished!');
}
}