pixelfed/app/Console/Commands/MediaS3GarbageCollector.php

<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use App\Media;
use App\Status;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use App\Services\MediaService;
use App\Services\StatusService;
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;

class MediaS3GarbageCollector extends Command
{
    /**
    * The name and signature of the console command.
    *
    * @var string
    */
    protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';

    /**
    * The console command description.
    *
    * @var string
    */
    protected $description = 'Delete (local) media uploads that exist on S3';

    /**
    * Create a new command instance.
    *
    * @return void
    */
    public function __construct()
    {
        parent::__construct();
    }

    /**
    * Execute the console command.
    *
    * @return int
    */
    public function handle()
    {
        $enabled = (bool) config_cache('pixelfed.cloud_storage');
        if(!$enabled) {
            $this->error('Cloud storage not enabled. Exiting...');
            return;
        }

        $deleteEnabled = config('media.delete_local_after_cloud');
        if(!$deleteEnabled) {
            $this->error('Delete local storage after cloud upload is not enabled');
            return;
        }

        $limit = $this->option('limit');
        $hugeMode = $this->option('huge');
        $log = $this->option('log-errors');

        if($limit > 2000 && !$hugeMode) {
            $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
            return;
        }

        $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first();

        if(!$minId) {
        	return;
        } else {
        	$minId = $minId->id;
        }

        return $hugeMode ?
            $this->hugeMode($minId, $limit, $log) :
            $this->regularMode($minId, $limit, $log);
    }

    protected function regularMode($minId, $limit, $log)
    {
        $gc = Media::whereRemoteMedia(false)
            ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
            ->whereNot('version', '4')
            ->where('id', '<', $minId)
            ->inRandomOrder()
            ->take($limit)
            ->get();

        $totalSize = 0;
        $bar = $this->output->createProgressBar($gc->count());
        $bar->start();
        $cloudDisk = Storage::disk(config('filesystems.cloud'));
        $localDisk = Storage::disk('local');

        foreach($gc as $media) {
            try {
                if(
                    $cloudDisk->exists($media->media_path)
                ) {
                    if( $localDisk->exists($media->media_path)) {
                        $localDisk->delete($media->media_path);
                        $media->version = 4;
                        $media->save();
                        $totalSize = $totalSize + $media->size;
                        MediaService::del($media->status_id);
                        StatusService::del($media->status_id, false);
                        if($localDisk->exists($media->thumbnail_path)) {
                            $localDisk->delete($media->thumbnail_path);
                        }
                    } else {
                        $media->version = 4;
                        $media->save();
                    }
                } else {
                    if($log) {
                        Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
                    }
                }
                $bar->advance();
            } catch (FileNotFoundException $e) {
                $bar->advance();
                continue;
            } catch (NotFoundHttpException $e) {
                $bar->advance();
                continue;
            } catch (\Exception $e) {
                $bar->advance();
                continue;
            }
        }
        $bar->finish();
        $this->line(' ');
        $this->info('Finished!');
        if($totalSize) {
            $this->info('Cleared ' . $totalSize . ' bytes of media from local disk!');
        }
        return 0;
    }

    protected function hugeMode($minId, $limit, $log)
    {
        $cloudDisk = Storage::disk(config('filesystems.cloud'));
        $localDisk = Storage::disk('local');

        $bar = $this->output->createProgressBar($limit);
        $bar->start();

        Media::whereRemoteMedia(false)
            ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
            ->whereNot('version', '4')
            ->where('id', '<', $minId)
            ->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
                foreach($medias as $media) {
                    try {
                        if($cloudDisk->exists($media->media_path)) {
                            if( $localDisk->exists($media->media_path)) {
                                $localDisk->delete($media->media_path);
                                $media->version = 4;
                                $media->save();
                                MediaService::del($media->status_id);
                                StatusService::del($media->status_id, false);
                                if($localDisk->exists($media->thumbnail_path)) {
                                    $localDisk->delete($media->thumbnail_path);
                                }
                            } else {
                                $media->version = 4;
                                $media->save();
                            }
                        } else {
                            if($log) {
                                Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
                            }
                        }
                        $bar->advance();
                    } catch (FileNotFoundException $e) {
                        $bar->advance();
                        continue;
                    } catch (NotFoundHttpException $e) {
                        $bar->advance();
                        continue;
                    } catch (\Exception $e) {
                        $bar->advance();
                        continue;
                    }
                }
        });

        $bar->finish();
        $this->line(' ');
        $this->info('Finished!');
    }
}