Merge pull request #3950 from pixelfed/staging

Staging

CHANGELOG.md

@@ -55,6 +55,7 @@
 - Update ApiV1Controller, remove min avatar size limit, fixes #3715 ([2b0db812](https://github.com/pixelfed/pixelfed/commit/2b0db812))
 - Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d))
 - Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494))
+- Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

app/Console/Commands/CatchUnoptimizedMedia.php

@@ -40,22 +40,20 @@ class CatchUnoptimizedMedia extends Command
      */
     public function handle()
     {
-        DB::transaction(function() {
-            Media::whereNull('processed_at')
-                ->where('skip_optimize', '!=', true)
-                ->whereNull('remote_url')
-                ->whereNotNull('status_id')
-                ->whereNotNull('media_path')
-                ->where('created_at', '>', now()->subHours(1))
-                ->whereIn('mime', [
-                    'image/jpeg',
-                    'image/png',
-                ])
-                ->chunk(50, function($medias) {
-                    foreach ($medias as $media) {
-                        ImageOptimize::dispatch($media);
-                    }
-                });
-         });
+        Media::whereNull('processed_at')
+            ->where('created_at', '>', now()->subHours(1))
+            ->where('skip_optimize', '!=', true)
+            ->whereNull('remote_url')
+            ->whereNotNull('status_id')
+            ->whereNotNull('media_path')
+            ->whereIn('mime', [
+                'image/jpeg',
+                'image/png',
+            ])
+            ->chunk(50, function($medias) {
+                foreach ($medias as $media) {
+                    ImageOptimize::dispatch($media);
+                }
+            });
     }
 }

app/Console/Commands/MediaGarbageCollector.php

@@ -4,7 +4,6 @@ namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
 use App\{Media, Status};
-use Carbon\Carbon;
 use App\Services\MediaStorageService;
 
 class MediaGarbageCollector extends Command
@@ -40,11 +39,10 @@ class MediaGarbageCollector extends Command
 	 */
 	public function handle()
 	{
-		$limit = 20000;
+		$limit = 500;
 
 		$gc = Media::whereNull('status_id')
-		->where('created_at', '<', Carbon::now()->subHours(12)->toDateTimeString())
-		->orderBy('created_at','asc')
+		->where('created_at', '<', now()->subHours(2)->toDateTimeString())
 		->take($limit)
 		->get();
 

app/Console/Commands/MediaS3GarbageCollector.php

@@ -0,0 +1,98 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Media;
+use App\Status;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+
+class MediaS3GarbageCollector extends Command
+{
+    /**
+    * The name and signature of the console command.
+    *
+    * @var string
+    */
+    protected $signature = 'media:s3gc {--limit=200}';
+
+    /**
+    * The console command description.
+    *
+    * @var string
+    */
+    protected $description = 'Delete (local) media uploads that exist on S3';
+
+    /**
+    * Create a new command instance.
+    *
+    * @return void
+    */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+    * Execute the console command.
+    *
+    * @return int
+    */
+    public function handle()
+    {
+        $enabled = config('pixelfed.cloud_storage');
+        if(!$enabled) {
+            $this->error('Cloud storage not enabled. Exiting...');
+            return;
+        }
+
+        $deleteEnabled = config('media.delete_local_after_cloud');
+        if(!$deleteEnabled) {
+            $this->error('Delete local storage after cloud upload is not enabled');
+            return;
+        }
+
+        $limit = $this->option('limit');
+        $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id;
+
+        $gc = Media::whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
+        ->whereNot('version', '4')
+        ->where('id', '<', $minId)
+        ->inRandomOrder()
+        ->take($limit)
+        ->get();
+
+        $totalSize = 0;
+        $bar = $this->output->createProgressBar($gc->count());
+        $bar->start();
+        $cloudDisk = Storage::disk(config('filesystems.cloud'));
+        $localDisk = Storage::disk('local');
+
+        foreach($gc as $media) {
+            if(
+                $cloudDisk->exists($media->media_path)
+            ) {
+                if( $localDisk->exists($media->media_path)) {
+                    $localDisk->delete($media->media_path);
+                    $media->version = 4;
+                    $media->save();
+                    $totalSize = $totalSize + $media->size;
+                } else {
+                    $media->version = 4;
+                    $media->save();
+                }
+            } else {
+                Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
+            }
+            $bar->advance();
+        }
+        $bar->finish();
+        $this->line(' ');
+        $this->info('Finished!');
+        if($totalSize) {
+            $this->info('Cleared ' . $totalSize . ' bytes of media from local disk!');
+        }
+        return 0;
+    }
+}

app/Console/Kernel.php

@@ -25,13 +25,17 @@ class Kernel extends ConsoleKernel
      */
     protected function schedule(Schedule $schedule)
     {
-        $schedule->command('media:optimize')->hourly();
-        $schedule->command('media:gc')->hourly();
+        $schedule->command('media:optimize')->hourlyAt(40);
+        $schedule->command('media:gc')->hourlyAt(5);
         $schedule->command('horizon:snapshot')->everyFiveMinutes();
         $schedule->command('story:gc')->everyFiveMinutes();
         $schedule->command('gc:failedjobs')->dailyAt(3);
         $schedule->command('gc:passwordreset')->dailyAt('09:41');
         $schedule->command('gc:sessions')->twiceDaily(13, 23);
+
+        if(config('pixelfed.cloud_storage') && config('media.delete_local_after_cloud')) {
+            $schedule->command('media:s3gc')->hourlyAt(15);
+        }
     }
 
     /**

config/logging.php

@@ -89,6 +89,12 @@ return [
         'emergency' => [
             'path' => storage_path('logs/laravel.log'),
         ],
+
+        'media' => [
+            'driver' => 'single',
+            'path'   => storage_path('logs/media.log'),
+            'level'  => 'debug'
+        ],
     ],
 
 ];

config/media.php

@@ -1,6 +1,8 @@
 <?php
 
 return [
+	'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
+
 	'exif' => [
 		'database' => env('MEDIA_EXIF_DATABASE', false),
 	],

database/migrations/2022_12_18_012352_add_status_id_index_to_media_table.php

@@ -0,0 +1,36 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('media', function (Blueprint $table) {
+            $table->index('status_id');
+            $table->index('replicated_at');
+            $table->index('version');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('media', function (Blueprint $table) {
+            $table->dropIndex('media_status_id_index');
+            $table->dropIndex('media_replicated_at_index');
+            $table->dropIndex('media_version_index');
+        });
+    }
+};