Merge pull request #4891 from pixelfed/staging

Add S3 IG Import Media Storage

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac))
 - Added Parental Controls ([#4862](https://github.com/pixelfed/pixelfed/pull/4862)) ([c91f1c59](https://github.com/pixelfed/pixelfed/commit/c91f1c59))
 - Added Forgot Email Feature ([67c650b1](https://github.com/pixelfed/pixelfed/commit/67c650b1))
+- Added S3 IG Import Media Storage support ([#4891](https://github.com/pixelfed/pixelfed/pull/4891)) ([081360b9](https://github.com/pixelfed/pixelfed/commit/081360b9))
 
 ### Federation
 - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))

app/Console/Commands/ImportUploadMediaToCloudStorage.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Models\ImportPost;
+use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline;
+use function Laravel\Prompts\progress;
+
+class ImportUploadMediaToCloudStorage extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import-upload-media-to-cloud-storage {--limit=500}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Migrate media imported from Instagram to S3 cloud storage.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        if(
+            (bool) config('import.instagram.storage.cloud.enabled') === false ||
+            (bool) config_cache('pixelfed.cloud_storage') === false
+        ) {
+            $this->error('Aborted. Cloud storage is not enabled for IG imports.');
+            return;
+        }
+
+        $limit = $this->option('limit');
+
+        $progress = progress(label: 'Migrating import media', steps: $limit);
+
+        $progress->start();
+
+        $posts = ImportPost::whereUploadedToS3(false)->take($limit)->get();
+
+        foreach($posts as $post) {
+            ImportMediaToCloudPipeline::dispatch($post)->onQueue('low');
+            $progress->advance();
+        }
+
+        $progress->finish();
+    }
+}

app/Console/Kernel.php

@@ -42,6 +42,10 @@ class Kernel extends ConsoleKernel
             $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51);
             $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
             $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32);
+
+            if(config('import.instagram.storage.cloud.enabled') && (bool) config_cache('pixelfed.cloud_storage')) {
+                $schedule->command('app:import-upload-media-to-cloud-storage')->hourlyAt(39);
+            }
         }
         $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
         $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);

app/Jobs/ImportPipeline/ImportMediaToCloudPipeline.php

@@ -0,0 +1,129 @@
+<?php
+
+namespace App\Jobs\ImportPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Models\ImportPost;
+use App\Media;
+use App\Services\MediaStorageService;
+use Illuminate\Support\Facades\Storage;
+use App\Jobs\VideoPipeline\VideoThumbnailToCloudPipeline;
+
+class ImportMediaToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $importPost;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'import-media-to-cloud-pipeline:ip-id:' . $this->importPost->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+    * Delete the job if its models no longer exist.
+    *
+    * @var bool
+    */
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(ImportPost $importPost)
+    {
+        $this->importPost = $importPost;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ip = $this->importPost;
+
+        if(
+            $ip->status_id === null ||
+            $ip->uploaded_to_s3 === true ||
+            (bool) config_cache('pixelfed.cloud_storage') === false) {
+            return;
+        }
+
+        $media = Media::whereStatusId($ip->status_id)->get();
+
+        if(!$media || !$media->count()) {
+            $importPost = ImportPost::find($ip->id);
+            $importPost->uploaded_to_s3 = true;
+            $importPost->save();
+            return;
+        }
+
+        foreach($media as $mediaPart) {
+            $this->handleMedia($mediaPart);
+        }
+    }
+
+    protected function handleMedia($media)
+    {
+        $ip = $this->importPost;
+
+        $importPost = ImportPost::find($ip->id);
+
+        if(!$importPost) {
+            return;
+        }
+
+        $res = MediaStorageService::move($media);
+
+        $importPost->uploaded_to_s3 = true;
+        $importPost->save();
+
+        if(!$res) {
+            return;
+        }
+
+        if($res === 'invalid file') {
+            return;
+        }
+
+        if($res === 'success') {
+            if($media->mime === 'video/mp4') {
+                VideoThumbnailToCloudPipeline::dispatch($media)->onQueue('low');
+            } else {
+                Storage::disk('local')->delete($media->media_path);
+            }
+        }
+    }
+}

app/Jobs/VideoPipeline/VideoThumbnailToCloudPipeline.php

@@ -0,0 +1,147 @@
+<?php
+
+namespace App\Jobs\VideoPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Http\File;
+use Cache;
+use FFMpeg;
+use Storage;
+use App\Media;
+use App\Jobs\MediaPipeline\MediaStoragePipeline;
+use App\Util\Media\Blurhash;
+use App\Services\MediaService;
+use App\Services\StatusService;
+use App\Services\ResilientMediaStorageService;
+
+class VideoThumbnailToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $media;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-thumb-to-cloud:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-thumb-to-cloud:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Media $media)
+    {
+        $this->media = $media;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if((bool) config_cache('pixelfed.cloud_storage') === false) {
+            return;
+        }
+
+        $media = $this->media;
+
+        if($media->mime != 'video/mp4') {
+            return;
+        }
+
+        if($media->profile_id === null || $media->status_id === null) {
+            return;
+        }
+
+        if($media->thumbnail_url) {
+            return;
+        }
+
+        $base = $media->media_path;
+        $path = explode('/', $base);
+        $name = last($path);
+
+        try {
+            $t = explode('.', $name);
+            $t = $t[0].'_thumb.jpeg';
+            $i = count($path) - 1;
+            $path[$i] = $t;
+            $save = implode('/', $path);
+            $video = FFMpeg::open($base)
+            ->getFrameFromSeconds(1)
+            ->export()
+            ->toDisk('local')
+            ->save($save);
+
+            if(!$save) {
+                return;
+            }
+
+            $media->thumbnail_path = $save;
+            $p = explode('/', $media->media_path);
+            array_pop($p);
+            $pt = explode('/', $save);
+            $thumbname = array_pop($pt);
+            $storagePath = implode('/', $p);
+            $thumb = storage_path('app/' . $save);
+            $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
+            $media->thumbnail_url = $thumbUrl;
+            $media->save();
+
+            $blurhash = Blurhash::generate($media);
+            if($blurhash) {
+                $media->blurhash = $blurhash;
+                $media->save();
+            }
+
+            if(str_starts_with($save, 'public/m/_v2/') && str_ends_with($save, '.jpeg')) {
+                Storage::delete($save);
+            }
+
+            if(str_starts_with($media->media_path, 'public/m/_v2/') && str_ends_with($media->media_path, '.mp4')) {
+                Storage::disk('local')->delete($media->media_path);
+            }
+        } catch (Exception $e) {
+        }
+
+        if($media->status_id) {
+            Cache::forget('status:transformer:media:attachments:' . $media->status_id);
+            MediaService::del($media->status_id);
+            Cache::forget('status:thumb:nsfw0' . $media->status_id);
+            Cache::forget('status:thumb:nsfw1' . $media->status_id);
+            Cache::forget('pf:services:sh:id:' . $media->status_id);
+            StatusService::del($media->status_id);
+        }
+    }
+}

app/Services/MediaStorageService.php

@@ -30,6 +30,18 @@ class MediaStorageService {
         return;
     }
 
+    public static function move(Media $media)
+    {
+        if($media->remote_media) {
+            return;
+        }
+
+        if(config_cache('pixelfed.cloud_storage') == true) {
+            return (new self())->cloudMove($media);
+        }
+        return;
+    }
+
     public static function avatar($avatar, $local = false, $skipRecentCheck = false)
     {
         return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
@@ -275,4 +287,41 @@ class MediaStorageService {
         }
         MediaDeletePipeline::dispatch($media)->onQueue('mmo');
     }
+
+    protected function cloudMove($media)
+    {
+        if(!Storage::exists($media->media_path)) {
+            return 'invalid file';
+        }
+
+        $path = storage_path('app/'.$media->media_path);
+        $thumb = false;
+        if($media->thumbnail_path) {
+            $thumb = storage_path('app/'.$media->thumbnail_path);
+            $pt = explode('/', $media->thumbnail_path);
+            $thumbname = array_pop($pt);
+        }
+
+        $p = explode('/', $media->media_path);
+        $name = array_pop($p);
+        $storagePath = implode('/', $p);
+
+        $url = ResilientMediaStorageService::store($storagePath, $path, $name);
+        if($thumb) {
+            $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
+            $media->thumbnail_url = $thumbUrl;
+        }
+        $media->cdn_url = $url;
+        $media->optimized_url = $url;
+        $media->replicated_at = now();
+        $media->save();
+
+        if($media->status_id) {
+            Cache::forget('status:transformer:media:attachments:' . $media->status_id);
+            MediaService::del($media->status_id);
+            StatusService::del($media->status_id, false);
+        }
+
+        return 'success';
+    }
 }

config/import.php

@@ -39,6 +39,12 @@ return [
 
             // Limit to specific user ids, in comma separated format
             'user_ids' => env('PF_IMPORT_IG_PERM_ONLY_USER_IDS', null),
+        ],
+
+        'storage' => [
+            'cloud' => [
+                'enabled' => env('PF_IMPORT_IG_CLOUD_STORAGE', env('PF_ENABLE_CLOUD', false)),
+            ]
         ]
     ]
 ];