Add import scheduler

app/Console/Commands/ImportUploadGarbageCollection.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Models\ImportPost;
+use Storage;
+use App\Services\ImportService;
+
+class ImportUploadGarbageCollection extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import-upload-garbage-collection';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        if(!config('import.instagram.enabled')) {
+            return;
+        }
+
+        $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(true)->take(100)->get();
+
+        if(!$ips->count()) {
+            return;
+        }
+
+        foreach($ips as $ip) {
+            $pid = $ip->profile_id;
+            $ip->delete();
+            ImportService::getPostCount($pid, true);
+            ImportService::clearAttempts($pid);
+            ImportService::getImportedFiles($pid, true);
+        }
+    }
+}

app/Console/Commands/TransformImports.php

@@ -0,0 +1,122 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Models\ImportPost;
+use App\Services\ImportService;
+use App\Media;
+use App\Profile;
+use App\Status;
+use Storage;
+use App\Services\MediaPathService;
+use Illuminate\Support\Str;
+use App\Util\Lexer\Autolink;
+
+class TransformImports extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:transform-imports';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Transform imports into statuses';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        if(!config('import.instagram.enabled')) {
+            return;
+        }
+
+        $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(10)->get();
+
+        if(!$ips->count()) {
+            return;
+        }
+
+        foreach($ips as $ip) {
+            $id = $ip->user_id;
+            $pid = $ip->profile_id;
+            $profile = Profile::find($pid);
+            $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day);
+
+            if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) {
+                ImportService::clearAttempts($profile->id);
+                ImportService::getPostCount($profile->id, true);
+                $ip->skip_missing_media = true;
+                $ip->save();
+                continue;
+            }
+
+            $missingMedia = false;
+            foreach($ip->media as $ipm) {
+                $fileName = last(explode('/', $ipm['uri']));
+                $og = 'imports/' . $id . '/' . $fileName;
+                if(!Storage::exists($og)) {
+                    $missingMedia = true;
+                }
+            }
+
+            if($missingMedia === true) {
+                $ip->skip_missing_media = true;
+                $ip->save();
+                continue;
+            }
+
+            $caption = $ip->caption;
+            $status = new Status;
+            $status->profile_id = $pid;
+            $status->caption = $caption;
+            $status->rendered = strlen(trim($caption)) ? Autolink::create()->autolink($ip->caption) : null;
+            $status->type = $ip->post_type;
+
+            $status->scope = 'unlisted';
+            $status->visibility = 'unlisted';
+            $status->id = $idk['id'];
+            $status->created_at = now()->parse($ip->creation_date);
+            $status->save();
+
+            foreach($ip->media as $ipm) {
+                $fileName = last(explode('/', $ipm['uri']));
+                $ext = last(explode('.', $fileName));
+                $basePath = MediaPathService::get($profile);
+                $og = 'imports/' . $id . '/' . $fileName;
+                if(!Storage::exists($og)) {
+                    $ip->skip_missing_media = true;
+                    $ip->save();
+                    continue;
+                }
+                $size = Storage::size($og);
+                $mime = Storage::mimeType($og);
+                $newFile = Str::random(40) . '.' . $ext;
+                $np = $basePath . '/' . $newFile;
+                Storage::move($og, $np);
+                $media = new Media;
+                $media->profile_id = $pid;
+                $media->user_id = $id;
+                $media->status_id = $status->id;
+                $media->media_path = $np;
+                $media->mime = $mime;
+                $media->size = $size;
+                $media->save();
+            }
+
+            $ip->status_id = $status->id;
+            $ip->creation_id = $idk['incr'];
+            $ip->save();
+
+            ImportService::clearAttempts($profile->id);
+            ImportService::getPostCount($profile->id, true);
+        }
+    }
+}

app/Console/Kernel.php

@@ -36,6 +36,11 @@ class Kernel extends ConsoleKernel
         if(in_array(config_cache('pixelfed.cloud_storage'), ['1', true, 'true']) && config('media.delete_local_after_cloud')) {
             $schedule->command('media:s3gc')->hourlyAt(15);
         }
+
+        if(config('import.instagram.enabled')) {
+            $schedule->command('app:transform-imports')->everyFourMinutes();
+            $schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes();
+        }
     }
 
     /**