Merge pull request #4505 from pixelfed/staging

Update IG Import commands, fix stalled import queue

CHANGELOG.md

@@ -17,6 +17,10 @@
 - Update AdminApiController, include more data for getUser method ([4f850e54](https://github.com/pixelfed/pixelfed/commit/4f850e54))
 - Update AdminApiController, improve admin moderation tools ([763ce19a](https://github.com/pixelfed/pixelfed/commit/763ce19a))
 - Update ActivityPubFetchService, fix authorized_fetch compatibility. Closes #1850, #2713, #2935 ([63a7879c](https://github.com/pixelfed/pixelfed/commit/63a7879c))
+- Update IG Import commands, fix stalled import queue ([b18f3fba](https://github.com/pixelfed/pixelfed/commit/b18f3fba))
+- Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts ([892907d5](https://github.com/pixelfed/pixelfed/commit/892907d5))
+- Update console kernel, add import upload gc ([afe6948d](https://github.com/pixelfed/pixelfed/commit/afe6948d))
+- Update ImportService, filter deleted posts from getImportedPosts endpoint ([10dd348c](https://github.com/pixelfed/pixelfed/commit/10dd348c))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8)

app/Console/Commands/ImportRemoveDeletedAccounts.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+use App\User;
+use App\Models\ImportPost;
+use App\Services\ImportService;
+
+class ImportRemoveDeletedAccounts extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import-remove-deleted-accounts';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    const CACHE_KEY = 'pf:services:import:gc-accounts:skip_min_id';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $skipMinId = Cache::remember(self::CACHE_KEY, 864000, function() {
+            return 1;
+        });
+
+        $deletedIds = User::withTrashed()
+            ->whereNotNull('status')
+            ->whereIn('status', ['deleted', 'delete'])
+            ->where('id', '>', $skipMinId)
+            ->limit(500)
+            ->pluck('id');
+
+        if(!$deletedIds || !$deletedIds->count()) {
+            return;
+        }
+
+        foreach($deletedIds as $did) {
+            if(Storage::exists('imports/' . $did)) {
+                Storage::deleteDirectory('imports/' . $did);
+            }
+
+            ImportPost::where('user_id', $did)->delete();
+            $skipMinId = $did;
+        }
+
+        Cache::put(self::CACHE_KEY, $skipMinId, 864000);
+    }
+}

app/Console/Commands/ImportUploadCleanStorage.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Models\ImportPost;
+use Storage;
+use App\Services\ImportService;
+use App\User;
+
+class ImportUploadCleanStorage extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import-upload-clean-storage';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $dirs = Storage::allDirectories('imports');
+
+        foreach($dirs as $dir) {
+            $uid = last(explode('/', $dir));
+            $skip = User::whereNull('status')->find($uid);
+            if(!$skip) {
+                Storage::deleteDirectory($dir);
+            }
+        }
+    }
+}

app/Console/Commands/ImportUploadGarbageCollection.php

@@ -32,7 +32,7 @@ class ImportUploadGarbageCollection extends Command
             return;
         }
 
-        $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(true)->take(100)->get();
+        $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', true)->take(100)->get();
 
         if(!$ips->count()) {
             return;

app/Console/Commands/TransformImports.php

@@ -38,7 +38,7 @@ class TransformImports extends Command
             return;
         }
 
-        $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get();
+        $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get();
 
         if(!$ips->count()) {
             return;
@@ -48,6 +48,26 @@ class TransformImports extends Command
             $id = $ip->user_id;
             $pid = $ip->profile_id;
             $profile = Profile::find($pid);
+            if(!$profile) {
+                $ip->skip_missing_media = true;
+                $ip->save();
+                continue;
+            }
+
+            $exists = ImportPost::whereUserId($id)
+                ->whereNotNull('status_id')
+                ->where('filename', $ip->filename)
+                ->where('creation_year', $ip->creation_year)
+                ->where('creation_month', $ip->creation_month)
+                ->where('creation_day', $ip->creation_day)
+                ->exists();
+
+            if($exists == true) {
+                $ip->skip_missing_media = true;
+                $ip->save();
+                continue;
+            }
+
             $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day);
 
             if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) {

app/Console/Kernel.php

@@ -39,7 +39,9 @@ class Kernel extends ConsoleKernel
 
         if(config('import.instagram.enabled')) {
             $schedule->command('app:transform-imports')->everyFourMinutes();
-            $schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes();
+            $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51);
+            $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
+            $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32);
         }
     }
 

app/Http/Controllers/ImportPostController.php

@@ -78,7 +78,7 @@ class ImportPostController extends Controller
 
         return ImportStatus::collection(
             ImportPost::whereProfileId($request->user()->profile_id)
-                ->whereNotNull('status_id')
+                ->has('status')
                 ->cursorPaginate(9)
         );
     }

app/Models/ImportPost.php

@@ -4,6 +4,7 @@ namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
+use App\Status;
 
 class ImportPost extends Model
 {
@@ -14,4 +15,9 @@ class ImportPost extends Model
         'creation_date' => 'datetime',
         'metadata' => 'json'
     ];
+
+    public function status()
+    {
+        return $this->hasOne(Status::class, 'id', 'status_id');
+    }
 }

app/Observers/StatusObserver.php

@@ -5,6 +5,8 @@ namespace App\Observers;
 use App\Status;
 use App\Services\ProfileStatusService;
 use Cache;
+use App\Models\ImportPost;
+use App\Services\ImportService;
 
 class StatusObserver
 {
@@ -56,6 +58,11 @@ class StatusObserver
         }
 
         ProfileStatusService::delete($status->profile_id, $status->id);
+
+        if($status->uri == null) {
+            ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete();
+            ImportService::clearImportedFiles($status->profile_id);
+        }
     }
 
     /**

app/Services/ImportService.php

@@ -97,9 +97,18 @@ class ImportService
         return Cache::remember($key, 21600, function() use($profileId) {
             return ImportPost::whereProfileId($profileId)
                 ->get()
+                ->filter(function($ip) {
+                    return StatusService::get($ip->status_id) == null;
+                })
                 ->map(function($ip) {
                     return collect($ip->media)->map(function($m) { return $m['uri']; });
-                })->flatten();
+                })->values()->flatten();
         });
     }
+
+    public static function clearImportedFiles($profileId)
+    {
+        $key = self::CACHE_KEY . 'importedPostsByProfileId:' . $profileId;
+        return Cache::forget($key);
+    }
 }