Update IG Import commands, fix stalled import queue

This commit is contained in:
Daniel Supernault 2023-06-26 03:21:38 -06:00
parent c64c4aa1cb
commit b18f3fba8b
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
3 changed files with 84 additions and 2 deletions

View file

@ -0,0 +1,61 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Storage;
use App\User;
use App\Models\ImportPost;
use App\Services\ImportService;
class ImportRemoveDeletedAccounts extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:import-remove-deleted-accounts';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
const CACHE_KEY = 'pf:services:import:gc-accounts:skip_min_id';
/**
* Execute the console command.
*/
public function handle()
{
$skipMinId = Cache::remember(self::CACHE_KEY, 864000, function() {
return 1;
});
$deletedIds = User::withTrashed()
->whereNotNull('status')
->whereIn('status', ['deleted', 'delete'])
->where('id', '>', $skipMinId)
->limit(500)
->pluck('id');
if(!$deletedIds || !$deletedIds->count()) {
return;
}
foreach($deletedIds as $did) {
if(Storage::exists('imports/' . $did)) {
Storage::deleteDirectory('imports/' . $did);
}
ImportPost::where('user_id', $did)->delete();
$skipMinId = $did;
}
Cache::put(self::CACHE_KEY, $skipMinId, 864000);
}
}

View file

@ -38,7 +38,7 @@ class TransformImports extends Command
return;
}
$ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get();
$ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get();
if(!$ips->count()) {
return;
@ -48,7 +48,27 @@ class TransformImports extends Command
$id = $ip->user_id;
$pid = $ip->profile_id;
$profile = Profile::find($pid);
if(!$profile) {
$ip->skip_missing_media = true;
$ip->save();
continue;
}
$idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day);
$exists = ImportPost::whereUserId($id)->where('filename', $ip->filename)->first();
if($exists) {
$cYear = str_pad($exists->creation_year, 2, 0, STR_PAD_LEFT);
$cMonth = str_pad($exists->creation_month, 2, 0, STR_PAD_LEFT);
$cDay = str_pad($exists->creation_day, 2, 0, STR_PAD_LEFT);
if( $cYear == $idk['year'] &&
$cMonth == $idk['month'] &&
$cDay == $idk['day']
) {
$ip->skip_missing_media = true;
$ip->save();
continue;
}
}
if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) {
ImportService::clearAttempts($profile->id);

View file

@ -39,7 +39,8 @@ class Kernel extends ConsoleKernel
if(config('import.instagram.enabled')) {
$schedule->command('app:transform-imports')->everyFourMinutes();
$schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes();
$schedule->command('app:import-upload-garbage-collection')->hourlyAt(51);
$schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
}
}