Update IG Import commands, fix stalled import queue

app/Console/Commands/ImportRemoveDeletedAccounts.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+use App\User;
+use App\Models\ImportPost;
+use App\Services\ImportService;
+
+class ImportRemoveDeletedAccounts extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import-remove-deleted-accounts';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    const CACHE_KEY = 'pf:services:import:gc-accounts:skip_min_id';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $skipMinId = Cache::remember(self::CACHE_KEY, 864000, function() {
+            return 1;
+        });
+
+        $deletedIds = User::withTrashed()
+            ->whereNotNull('status')
+            ->whereIn('status', ['deleted', 'delete'])
+            ->where('id', '>', $skipMinId)
+            ->limit(500)
+            ->pluck('id');
+
+        if(!$deletedIds || !$deletedIds->count()) {
+            return;
+        }
+
+        foreach($deletedIds as $did) {
+            if(Storage::exists('imports/' . $did)) {
+                Storage::deleteDirectory('imports/' . $did);
+            }
+
+            ImportPost::where('user_id', $did)->delete();
+            $skipMinId = $did;
+        }
+
+        Cache::put(self::CACHE_KEY, $skipMinId, 864000);
+    }
+}

app/Console/Commands/TransformImports.php

@@ -38,7 +38,7 @@ class TransformImports extends Command
             return;
         }
 
-        $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get();
+        $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get();
 
         if(!$ips->count()) {
             return;
@@ -48,7 +48,27 @@ class TransformImports extends Command
             $id = $ip->user_id;
             $pid = $ip->profile_id;
             $profile = Profile::find($pid);
+            if(!$profile) {
+                $ip->skip_missing_media = true;
+                $ip->save();
+                continue;
+            }
+
             $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day);
+            $exists = ImportPost::whereUserId($id)->where('filename', $ip->filename)->first();
+            if($exists) {
+                $cYear = str_pad($exists->creation_year, 2, 0, STR_PAD_LEFT);
+                $cMonth = str_pad($exists->creation_month, 2, 0, STR_PAD_LEFT);
+                $cDay = str_pad($exists->creation_day, 2, 0, STR_PAD_LEFT);
+                if( $cYear == $idk['year'] &&
+                    $cMonth == $idk['month'] &&
+                    $cDay == $idk['day']
+                ) {
+                    $ip->skip_missing_media = true;
+                    $ip->save();
+                    continue;
+                }
+            }
 
             if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) {
                 ImportService::clearAttempts($profile->id);

app/Console/Kernel.php

@@ -39,7 +39,8 @@ class Kernel extends ConsoleKernel
 
         if(config('import.instagram.enabled')) {
             $schedule->command('app:transform-imports')->everyFourMinutes();
-            $schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes();
+            $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51);
+            $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
         }
     }