From b18f3fba8b33d542d165493018b2d62372492422 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 03:21:38 -0600 Subject: [PATCH 1/9] Update IG Import commands, fix stalled import queue --- .../Commands/ImportRemoveDeletedAccounts.php | 61 +++++++++++++++++++ app/Console/Commands/TransformImports.php | 22 ++++++- app/Console/Kernel.php | 3 +- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 app/Console/Commands/ImportRemoveDeletedAccounts.php diff --git a/app/Console/Commands/ImportRemoveDeletedAccounts.php b/app/Console/Commands/ImportRemoveDeletedAccounts.php new file mode 100644 index 000000000..e1e6acafd --- /dev/null +++ b/app/Console/Commands/ImportRemoveDeletedAccounts.php @@ -0,0 +1,61 @@ +whereNotNull('status') + ->whereIn('status', ['deleted', 'delete']) + ->where('id', '>', $skipMinId) + ->limit(500) + ->pluck('id'); + + if(!$deletedIds || !$deletedIds->count()) { + return; + } + + foreach($deletedIds as $did) { + if(Storage::exists('imports/' . $did)) { + Storage::deleteDirectory('imports/' . $did); + } + + ImportPost::where('user_id', $did)->delete(); + $skipMinId = $did; + } + + Cache::put(self::CACHE_KEY, $skipMinId, 864000); + } +} diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index a08f7ed76..3b9509387 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -38,7 +38,7 @@ class TransformImports extends Command return; } - $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get(); + $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get(); if(!$ips->count()) { return; @@ -48,7 +48,27 @@ class TransformImports extends Command $id = $ip->user_id; $pid = $ip->profile_id; $profile = Profile::find($pid); + if(!$profile) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } + $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); + $exists = ImportPost::whereUserId($id)->where('filename', $ip->filename)->first(); + if($exists) { + $cYear = str_pad($exists->creation_year, 2, 0, STR_PAD_LEFT); + $cMonth = str_pad($exists->creation_month, 2, 0, STR_PAD_LEFT); + $cDay = str_pad($exists->creation_day, 2, 0, STR_PAD_LEFT); + if( $cYear == $idk['year'] && + $cMonth == $idk['month'] && + $cDay == $idk['day'] + ) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } + } if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) { ImportService::clearAttempts($profile->id); diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0fb20f630..7b1f632ed 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -39,7 +39,8 @@ class Kernel extends ConsoleKernel if(config('import.instagram.enabled')) { $schedule->command('app:transform-imports')->everyFourMinutes(); - $schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes(); + $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51); + $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37); } } From 892907d5d18a5b78661d001e5b7ebfa3189b0c94 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:16:49 -0600 Subject: [PATCH 2/9] Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts --- .../ImportUploadGarbageCollection.php | 2 +- app/Console/Commands/TransformImports.php | 28 +++++++++---------- app/Observers/StatusObserver.php | 5 ++++ app/Services/ImportService.php | 6 ++++ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/app/Console/Commands/ImportUploadGarbageCollection.php b/app/Console/Commands/ImportUploadGarbageCollection.php index cec323dcf..7d1ab24b5 100644 --- a/app/Console/Commands/ImportUploadGarbageCollection.php +++ b/app/Console/Commands/ImportUploadGarbageCollection.php @@ -32,7 +32,7 @@ class ImportUploadGarbageCollection extends Command return; } - $ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(true)->take(100)->get(); + $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', true)->take(100)->get(); if(!$ips->count()) { return; diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index 3b9509387..cd63985ac 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -54,22 +54,22 @@ class TransformImports extends Command continue; } - $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); - $exists = ImportPost::whereUserId($id)->where('filename', $ip->filename)->first(); - if($exists) { - $cYear = str_pad($exists->creation_year, 2, 0, STR_PAD_LEFT); - $cMonth = str_pad($exists->creation_month, 2, 0, STR_PAD_LEFT); - $cDay = str_pad($exists->creation_day, 2, 0, STR_PAD_LEFT); - if( $cYear == $idk['year'] && - $cMonth == $idk['month'] && - $cDay == $idk['day'] - ) { - $ip->skip_missing_media = true; - $ip->save(); - continue; - } + $exists = ImportPost::whereUserId($id) + ->whereNotNull('status_id') + ->where('filename', $ip->filename) + ->where('creation_year', $ip->creation_year) + ->where('creation_month', $ip->creation_month) + ->where('creation_day', $ip->creation_day) + ->exists(); + + if($exists == true) { + $ip->skip_missing_media = true; + $ip->save(); + continue; } + $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); + if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) { ImportService::clearAttempts($profile->id); ImportService::getPostCount($profile->id, true); diff --git a/app/Observers/StatusObserver.php b/app/Observers/StatusObserver.php index 8d370b38e..99d6a89ff 100644 --- a/app/Observers/StatusObserver.php +++ b/app/Observers/StatusObserver.php @@ -5,6 +5,7 @@ namespace App\Observers; use App\Status; use App\Services\ProfileStatusService; use Cache; +use App\Services\ImportService; class StatusObserver { @@ -56,6 +57,10 @@ class StatusObserver } ProfileStatusService::delete($status->profile_id, $status->id); + + if($status->uri == null) { + ImportService::clearImportedFiles($status->profile_id); + } } /** diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php index 2ed43e73f..8ca12b445 100644 --- a/app/Services/ImportService.php +++ b/app/Services/ImportService.php @@ -102,4 +102,10 @@ class ImportService })->flatten(); }); } + + public static function clearImportedFiles($profileId) + { + $key = self::CACHE_KEY . 'importedPostsByProfileId:' . $profileId; + return Cache::forget($key); + } } From b47e8f8e3e8c46d306edbb31e8170d2b495172ba Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:18:39 -0600 Subject: [PATCH 3/9] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c52910..d06ad189f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ - Update AdminApiController, include more data for getUser method ([4f850e54](https://github.com/pixelfed/pixelfed/commit/4f850e54)) - Update AdminApiController, improve admin moderation tools ([763ce19a](https://github.com/pixelfed/pixelfed/commit/763ce19a)) - Update ActivityPubFetchService, fix authorized_fetch compatibility. Closes #1850, #2713, #2935 ([63a7879c](https://github.com/pixelfed/pixelfed/commit/63a7879c)) +- Update IG Import commands, fix stalled import queue ([b18f3fba](https://github.com/pixelfed/pixelfed/commit/b18f3fba)) +- Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts ([892907d5](https://github.com/pixelfed/pixelfed/commit/892907d5)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) From afe6948da87ed4267c2cb0586d1407d81af49b0e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:33:47 -0600 Subject: [PATCH 4/9] Update console kernel, add import upload gc --- .../Commands/ImportUploadCleanStorage.php | 42 +++++++++++++++++++ app/Console/Kernel.php | 1 + 2 files changed, 43 insertions(+) create mode 100644 app/Console/Commands/ImportUploadCleanStorage.php diff --git a/app/Console/Commands/ImportUploadCleanStorage.php b/app/Console/Commands/ImportUploadCleanStorage.php new file mode 100644 index 000000000..31b0fac74 --- /dev/null +++ b/app/Console/Commands/ImportUploadCleanStorage.php @@ -0,0 +1,42 @@ +find($uid); + if(!$skip) { + Storage::deleteDirectory($dir); + } + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 7b1f632ed..046924eb6 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -41,6 +41,7 @@ class Kernel extends ConsoleKernel $schedule->command('app:transform-imports')->everyFourMinutes(); $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51); $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37); + $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32); } } From d6d60a8574b84ba2778b99fdf0a57be085470067 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:34:12 -0600 Subject: [PATCH 5/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d06ad189f..e12d63317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Update ActivityPubFetchService, fix authorized_fetch compatibility. Closes #1850, #2713, #2935 ([63a7879c](https://github.com/pixelfed/pixelfed/commit/63a7879c)) - Update IG Import commands, fix stalled import queue ([b18f3fba](https://github.com/pixelfed/pixelfed/commit/b18f3fba)) - Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts ([892907d5](https://github.com/pixelfed/pixelfed/commit/892907d5)) +- Update console kernel, add import upload gc ([afe6948d](https://github.com/pixelfed/pixelfed/commit/afe6948d)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) From 10dd348c28faab466b2c2cf14b05287a4c9f052c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:46:57 -0600 Subject: [PATCH 6/9] Update ImportService, filter deleted posts from getImportedPosts endpoint --- app/Observers/StatusObserver.php | 1 + app/Services/ImportService.php | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Observers/StatusObserver.php b/app/Observers/StatusObserver.php index 99d6a89ff..6cee10a97 100644 --- a/app/Observers/StatusObserver.php +++ b/app/Observers/StatusObserver.php @@ -59,6 +59,7 @@ class StatusObserver ProfileStatusService::delete($status->profile_id, $status->id); if($status->uri == null) { + ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete(); ImportService::clearImportedFiles($status->profile_id); } } diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php index 8ca12b445..aac3fde4a 100644 --- a/app/Services/ImportService.php +++ b/app/Services/ImportService.php @@ -97,9 +97,12 @@ class ImportService return Cache::remember($key, 21600, function() use($profileId) { return ImportPost::whereProfileId($profileId) ->get() + ->filter(function($ip) { + return StatusService::get($ip->status_id); + }) ->map(function($ip) { return collect($ip->media)->map(function($m) { return $m['uri']; }); - })->flatten(); + })->values()->flatten(); }); } From 6fd53a3001f5d5f441ac92cbccaf51db257e971d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 04:47:43 -0600 Subject: [PATCH 7/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e12d63317..73590f4ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Update IG Import commands, fix stalled import queue ([b18f3fba](https://github.com/pixelfed/pixelfed/commit/b18f3fba)) - Update TransformImports command, improve handling of imported posts that already exist or are from deleted accounts ([892907d5](https://github.com/pixelfed/pixelfed/commit/892907d5)) - Update console kernel, add import upload gc ([afe6948d](https://github.com/pixelfed/pixelfed/commit/afe6948d)) +- Update ImportService, filter deleted posts from getImportedPosts endpoint ([10dd348c](https://github.com/pixelfed/pixelfed/commit/10dd348c)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) From fe6123c8204132a9dd982063b75a237ec4bfec0b Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 05:38:29 -0600 Subject: [PATCH 8/9] Update ImportPostController --- app/Http/Controllers/ImportPostController.php | 2 +- app/Models/ImportPost.php | 6 ++++++ app/Observers/StatusObserver.php | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/ImportPostController.php b/app/Http/Controllers/ImportPostController.php index 3d1462812..e814c2b3a 100644 --- a/app/Http/Controllers/ImportPostController.php +++ b/app/Http/Controllers/ImportPostController.php @@ -78,7 +78,7 @@ class ImportPostController extends Controller return ImportStatus::collection( ImportPost::whereProfileId($request->user()->profile_id) - ->whereNotNull('status_id') + ->has('status') ->cursorPaginate(9) ); } diff --git a/app/Models/ImportPost.php b/app/Models/ImportPost.php index 075b63111..7fb37d4c7 100644 --- a/app/Models/ImportPost.php +++ b/app/Models/ImportPost.php @@ -4,6 +4,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; +use App\Status; class ImportPost extends Model { @@ -14,4 +15,9 @@ class ImportPost extends Model 'creation_date' => 'datetime', 'metadata' => 'json' ]; + + public function status() + { + return $this->hasOne(Status::class, 'id', 'status_id'); + } } diff --git a/app/Observers/StatusObserver.php b/app/Observers/StatusObserver.php index 6cee10a97..e58997165 100644 --- a/app/Observers/StatusObserver.php +++ b/app/Observers/StatusObserver.php @@ -5,6 +5,7 @@ namespace App\Observers; use App\Status; use App\Services\ProfileStatusService; use Cache; +use App\Models\ImportPost; use App\Services\ImportService; class StatusObserver From 2f2e446c1f1e99687d95ae1f18c573822baffdfc Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 26 Jun 2023 05:39:16 -0600 Subject: [PATCH 9/9] Update ImportService --- app/Services/ImportService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php index aac3fde4a..4f85dabbe 100644 --- a/app/Services/ImportService.php +++ b/app/Services/ImportService.php @@ -98,7 +98,7 @@ class ImportService return ImportPost::whereProfileId($profileId) ->get() ->filter(function($ip) { - return StatusService::get($ip->status_id); + return StatusService::get($ip->status_id) == null; }) ->map(function($ip) { return collect($ip->media)->map(function($m) { return $m['uri']; });