Add S3 IG Import Media Storage

This commit is contained in:
Daniel Supernault 2024-02-02 02:29:33 -07:00
parent 5b7111c56f
commit 622e9cee97
No known key found for this signature in database
GPG key ID: 23740873EE6F76A1
4 changed files with 431 additions and 198 deletions

View file

@ -0,0 +1,54 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Models\ImportPost;
use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline;
use function Laravel\Prompts\progress;
class ImportUploadMediaToCloudStorage extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:import-upload-media-to-cloud-storage {--limit=500}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Migrate media imported from Instagram to S3 cloud storage.';
/**
* Execute the console command.
*/
public function handle()
{
if(
(bool) config('import.instagram.storage.cloud.enabled') === false ||
(bool) config_cache('pixelfed.cloud_storage') === false
) {
$this->error('Aborted. Cloud storage is not enabled for IG imports.');
return;
}
$limit = $this->option('limit');
$progress = progress(label: 'Migrating import media', steps: $limit);
$progress->start();
$posts = ImportPost::whereUploadedToS3(false)->take($limit)->get();
foreach($posts as $post) {
ImportMediaToCloudPipeline::dispatch($post)->onQueue('low');
$progress->advance();
}
$progress->finish();
}
}

View file

@ -0,0 +1,124 @@
<?php
namespace App\Jobs\ImportPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Models\ImportPost;
use App\Media;
use App\Services\MediaStorageService;
use Illuminate\Support\Facades\Storage;
class ImportMediaToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $importPost;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'import-media-to-cloud-pipeline:ip-id:' . $this->importPost->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()];
}
/**
* Delete the job if its models no longer exist.
*
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Create a new job instance.
*/
public function __construct(ImportPost $importPost)
{
$this->importPost = $importPost;
}
/**
* Execute the job.
*/
public function handle(): void
{
$ip = $this->importPost;
if(
$ip->status_id === null ||
$ip->uploaded_to_s3 === true ||
(bool) config_cache('pixelfed.cloud_storage') === false) {
return;
}
$media = Media::whereStatusId($ip->status_id)->get();
if(!$media || !$media->count()) {
$importPost = ImportPost::find($ip->id);
$importPost->uploaded_to_s3 = true;
$importPost->save();
return;
}
foreach($media as $mediaPart) {
$this->handleMedia($mediaPart);
}
}
protected function handleMedia($media)
{
$ip = $this->importPost;
$importPost = ImportPost::find($ip->id);
if(!$importPost) {
return;
}
$res = MediaStorageService::move($media);
$importPost->uploaded_to_s3 = true;
$importPost->save();
if(!$res) {
return;
}
if($res === 'invalid file') {
return;
}
if($res === 'success') {
Storage::disk('local')->delete($media->media_path);
}
}
}

View file

@ -30,6 +30,18 @@ class MediaStorageService {
return;
}
public static function move(Media $media)
{
if($media->remote_media) {
return;
}
if(config_cache('pixelfed.cloud_storage') == true) {
return (new self())->cloudMove($media);
}
return;
}
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
{
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
@ -275,4 +287,41 @@ class MediaStorageService {
}
MediaDeletePipeline::dispatch($media)->onQueue('mmo');
}
protected function cloudMove($media)
{
if(!Storage::exists($media->media_path)) {
return 'invalid file';
}
$path = storage_path('app/'.$media->media_path);
$thumb = false;
if($media->thumbnail_path) {
$thumb = storage_path('app/'.$media->thumbnail_path);
$pt = explode('/', $media->thumbnail_path);
$thumbname = array_pop($pt);
}
$p = explode('/', $media->media_path);
$name = array_pop($p);
$storagePath = implode('/', $p);
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
if($thumb) {
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
$media->thumbnail_url = $thumbUrl;
}
$media->cdn_url = $url;
$media->optimized_url = $url;
$media->replicated_at = now();
$media->save();
if($media->status_id) {
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
MediaService::del($media->status_id);
StatusService::del($media->status_id, false);
}
return 'success';
}
}

View file

@ -39,6 +39,12 @@ return [
// Limit to specific user ids, in comma separated format
'user_ids' => env('PF_IMPORT_IG_PERM_ONLY_USER_IDS', null),
],
'storage' => [
'cloud' => [
'enabled' => env('PF_IMPORT_IG_CLOUD_STORAGE', env('PF_ENABLE_CLOUD', false)),
]
]
]
];