diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c52910..73590f4ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ - Update AdminApiController, include more data for getUser method ([4f850e54](https://github.com/pixelfed/pixelfed/commit/4f850e54)) - Update AdminApiController, improve admin moderation tools ([763ce19a](https://github.com/pixelfed/pixelfed/commit/763ce19a)) - Update ActivityPubFetchService, fix authorized_fetch compatibility. Closes #1850, #2713, #2935 ([63a7879c](https://github.com/pixelfed/pixelfed/commit/63a7879c)) +- Update IG Import commands, fix stalled import queue ([b18f3fba](https://github.com/pixelfed/pixelfed/commit/b18f3fba)) +- Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts ([892907d5](https://github.com/pixelfed/pixelfed/commit/892907d5)) +- Update console kernel, add import upload gc ([afe6948d](https://github.com/pixelfed/pixelfed/commit/afe6948d)) +- Update ImportService, filter deleted posts from getImportedPosts endpoint ([10dd348c](https://github.com/pixelfed/pixelfed/commit/10dd348c)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) diff --git a/app/Console/Commands/ImportRemoveDeletedAccounts.php b/app/Console/Commands/ImportRemoveDeletedAccounts.php new file mode 100644 index 000000000..e1e6acafd --- /dev/null +++ b/app/Console/Commands/ImportRemoveDeletedAccounts.php @@ -0,0 +1,61 @@ +whereNotNull('status') + ->whereIn('status', ['deleted', 'delete']) + ->where('id', '>', $skipMinId) + ->limit(500) + ->pluck('id'); + + if(!$deletedIds || !$deletedIds->count()) { + return; + } + + foreach($deletedIds as $did) { + if(Storage::exists('imports/' . $did)) { + Storage::deleteDirectory('imports/' . $did); + } + + ImportPost::where('user_id', $did)->delete(); + $skipMinId = $did; + } + + Cache::put(self::CACHE_KEY, $skipMinId, 864000); + } +} diff --git a/app/Console/Commands/ImportUploadCleanStorage.php b/app/Console/Commands/ImportUploadCleanStorage.php new file mode 100644 index 000000000..31b0fac74 --- /dev/null +++ b/app/Console/Commands/ImportUploadCleanStorage.php @@ -0,0 +1,42 @@ +find($uid); + if(!$skip) { + Storage::deleteDirectory($dir); + } + } + } +} diff --git a/app/Console/Commands/ImportUploadGarbageCollection.php b/app/Console/Commands/ImportUploadGarbageCollection.php index cec323dcf..7d1ab24b5 100644 --- a/app/Console/Commands/ImportUploadGarbageCollection.php +++ b/app/Console/Commands/ImportUploadGarbageCollection.php @@ -32,7 +32,7 @@ class ImportUploadGarbageCollection extends Command return; } - $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(true)->take(100)->get(); + $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', true)->take(100)->get(); if(!$ips->count()) { return; diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index a08f7ed76..cd63985ac 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -38,7 +38,7 @@ class TransformImports extends Command return; } - $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get(); + $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get(); if(!$ips->count()) { return; @@ -48,6 +48,26 @@ class TransformImports extends Command $id = $ip->user_id; $pid = $ip->profile_id; $profile = Profile::find($pid); + if(!$profile) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } + + $exists = ImportPost::whereUserId($id) + ->whereNotNull('status_id') + ->where('filename', $ip->filename) + ->where('creation_year', $ip->creation_year) + ->where('creation_month', $ip->creation_month) + ->where('creation_day', $ip->creation_day) + ->exists(); + + if($exists == true) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } + $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) { diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0fb20f630..046924eb6 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -39,7 +39,9 @@ class Kernel extends ConsoleKernel if(config('import.instagram.enabled')) { $schedule->command('app:transform-imports')->everyFourMinutes(); - $schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes(); + $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51); + $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37); + $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32); } } diff --git a/app/Http/Controllers/ImportPostController.php b/app/Http/Controllers/ImportPostController.php index 3d1462812..e814c2b3a 100644 --- a/app/Http/Controllers/ImportPostController.php +++ b/app/Http/Controllers/ImportPostController.php @@ -78,7 +78,7 @@ class ImportPostController extends Controller return ImportStatus::collection( ImportPost::whereProfileId($request->user()->profile_id) - ->whereNotNull('status_id') + ->has('status') ->cursorPaginate(9) ); } diff --git a/app/Models/ImportPost.php b/app/Models/ImportPost.php index 075b63111..7fb37d4c7 100644 --- a/app/Models/ImportPost.php +++ b/app/Models/ImportPost.php @@ -4,6 +4,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; +use App\Status; class ImportPost extends Model { @@ -14,4 +15,9 @@ class ImportPost extends Model 'creation_date' => 'datetime', 'metadata' => 'json' ]; + + public function status() + { + return $this->hasOne(Status::class, 'id', 'status_id'); + } } diff --git a/app/Observers/StatusObserver.php b/app/Observers/StatusObserver.php index 8d370b38e..e58997165 100644 --- a/app/Observers/StatusObserver.php +++ b/app/Observers/StatusObserver.php @@ -5,6 +5,8 @@ namespace App\Observers; use App\Status; use App\Services\ProfileStatusService; use Cache; +use App\Models\ImportPost; +use App\Services\ImportService; class StatusObserver { @@ -56,6 +58,11 @@ class StatusObserver } ProfileStatusService::delete($status->profile_id, $status->id); + + if($status->uri == null) { + ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete(); + ImportService::clearImportedFiles($status->profile_id); + } } /** diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php index 2ed43e73f..4f85dabbe 100644 --- a/app/Services/ImportService.php +++ b/app/Services/ImportService.php @@ -97,9 +97,18 @@ class ImportService return Cache::remember($key, 21600, function() use($profileId) { return ImportPost::whereProfileId($profileId) ->get() + ->filter(function($ip) { + return StatusService::get($ip->status_id) == null; + }) ->map(function($ip) { return collect($ip->media)->map(function($m) { return $m['uri']; }); - })->flatten(); + })->values()->flatten(); }); } + + public static function clearImportedFiles($profileId) + { + $key = self::CACHE_KEY . 'importedPostsByProfileId:' . $profileId; + return Cache::forget($key); + } }