diff --git a/CHANGELOG.md b/CHANGELOG.md index d251b3df3..d939ea1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac)) - Added Parental Controls ([#4862](https://github.com/pixelfed/pixelfed/pull/4862)) ([c91f1c59](https://github.com/pixelfed/pixelfed/commit/c91f1c59)) - Added Forgot Email Feature ([67c650b1](https://github.com/pixelfed/pixelfed/commit/67c650b1)) +- Added S3 IG Import Media Storage support ([#4891](https://github.com/pixelfed/pixelfed/pull/4891)) ([081360b9](https://github.com/pixelfed/pixelfed/commit/081360b9)) ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) @@ -89,6 +90,21 @@ - Update meta tags, improve descriptions and seo/og tags ([fd44c80c](https://github.com/pixelfed/pixelfed/commit/fd44c80c)) - Update login view, add email prefill logic ([d76f0168](https://github.com/pixelfed/pixelfed/commit/d76f0168)) - Update LoginController, fix captcha validation error message ([0325e171](https://github.com/pixelfed/pixelfed/commit/0325e171)) +- Update ApiV1Controller, properly cast boolean sensitive parameter. Fixes #4888 ([0aff126a](https://github.com/pixelfed/pixelfed/commit/0aff126a)) +- Update AccountImport.vue, fix new IG export format ([59aa6a4b](https://github.com/pixelfed/pixelfed/commit/59aa6a4b)) +- Update TransformImports command, fix import service condition ([32c59f04](https://github.com/pixelfed/pixelfed/commit/32c59f04)) +- Update AP helpers, more efficently update post count ([7caed381](https://github.com/pixelfed/pixelfed/commit/7caed381)) +- Update AP helpers, refactor post count decrement logic ([b81ae577](https://github.com/pixelfed/pixelfed/commit/b81ae577)) +- Update AP helpers, fix sensitive bug ([00ed330c](https://github.com/pixelfed/pixelfed/commit/00ed330c)) +- Update NotificationEpochUpdatePipeline, use more efficient query ([4d401389](https://github.com/pixelfed/pixelfed/commit/4d401389)) +- Update notification pipelines, fix non-local saving ([fa97a1f3](https://github.com/pixelfed/pixelfed/commit/fa97a1f3)) +- Update NodeinfoService, disable redirects ([240e6bbe](https://github.com/pixelfed/pixelfed/commit/240e6bbe)) +- Update Instance model, add entity casts ([289cad47](https://github.com/pixelfed/pixelfed/commit/289cad47)) +- Update FetchNodeinfoPipeline, use more efficient dispatch ([ac01f51a](https://github.com/pixelfed/pixelfed/commit/ac01f51a)) +- Update horizon.php config ([1e3acade](https://github.com/pixelfed/pixelfed/commit/1e3acade)) +- Update PublicApiController, consume InstanceService blocked domains for account and statuses endpoints ([01b33fb3](https://github.com/pixelfed/pixelfed/commit/01b33fb3)) +- Update ApiV1Controller, enforce blocked instance domain logic ([5b284cac](https://github.com/pixelfed/pixelfed/commit/5b284cac)) +- Update ApiV2Controller, add vapid key to instance object. Thanks thisismissem! ([4d02d6f1](https://github.com/pixelfed/pixelfed/commit/4d02d6f1)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) diff --git a/app/Console/Commands/AccountPostCountStatUpdate.php b/app/Console/Commands/AccountPostCountStatUpdate.php new file mode 100644 index 000000000..6d5ba00a6 --- /dev/null +++ b/app/Console/Commands/AccountPostCountStatUpdate.php @@ -0,0 +1,57 @@ +count(); + if($statusCount != $acct['statuses_count']) { + $profile = Profile::find($id); + if(!$profile) { + AccountStatService::removeFromPostCount($id); + continue; + } + $profile->status_count = $statusCount; + $profile->save(); + AccountService::del($id); + } + AccountStatService::removeFromPostCount($id); + } + return; + } +} diff --git a/app/Console/Commands/ImportUploadMediaToCloudStorage.php b/app/Console/Commands/ImportUploadMediaToCloudStorage.php new file mode 100644 index 000000000..bf23794c9 --- /dev/null +++ b/app/Console/Commands/ImportUploadMediaToCloudStorage.php @@ -0,0 +1,54 @@ +error('Aborted. Cloud storage is not enabled for IG imports.'); + return; + } + + $limit = $this->option('limit'); + + $progress = progress(label: 'Migrating import media', steps: $limit); + + $progress->start(); + + $posts = ImportPost::whereUploadedToS3(false)->take($limit)->get(); + + foreach($posts as $post) { + ImportMediaToCloudPipeline::dispatch($post)->onQueue('low'); + $progress->advance(); + } + + $progress->finish(); + } +} diff --git a/app/Console/Commands/InstanceManager.php b/app/Console/Commands/InstanceManager.php new file mode 100644 index 000000000..a495d9617 --- /dev/null +++ b/app/Console/Commands/InstanceManager.php @@ -0,0 +1,298 @@ +recalculateStats(); + break; + + case 'Unlisted Instances': + return $this->viewUnlistedInstances(); + break; + + case 'Banned Instances': + return $this->viewBannedInstances(); + break; + + case 'Unlist Instance': + return $this->unlistInstance(); + break; + + case 'Ban Instance': + return $this->banInstance(); + break; + + case 'Unban Instance': + return $this->unbanInstance(); + break; + + case 'Relist Instance': + return $this->relistInstance(); + break; + } + } + + protected function recalculateStats() + { + $instanceCount = Instance::count(); + $confirmed = confirm('Do you want to recalculate stats for all ' . $instanceCount . ' instances?'); + if(!$confirmed) { + $this->error('Aborting...'); + exit; + } + + $users = progress( + label: 'Updating instance stats...', + steps: Instance::all(), + callback: fn ($instance) => $this->updateInstanceStats($instance), + ); + } + + protected function updateInstanceStats($instance) + { + FetchNodeinfoPipeline::dispatch($instance)->onQueue('intbg'); + } + + protected function unlistInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereUnlisted(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to unlist this instance?'); + if(!$confirmed) { + $this->error('Aborting instance unlisting'); + exit; + } + + $instance->unlisted = true; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully unlisted ' . $instance->domain . '!'); + exit; + } + + protected function relistInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereUnlisted(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to re-list this instance?'); + if(!$confirmed) { + $this->error('Aborting instance re-listing'); + exit; + } + + $instance->unlisted = false; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully re-listed ' . $instance->domain . '!'); + exit; + } + + protected function banInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereBanned(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to ban this instance?'); + if(!$confirmed) { + $this->error('Aborting instance ban'); + exit; + } + + $instance->banned = true; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully banned ' . $instance->domain . '!'); + exit; + } + + protected function unbanInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereBanned(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to unban this instance?'); + if(!$confirmed) { + $this->error('Aborting instance unban'); + exit; + } + + $instance->banned = false; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully un-banned ' . $instance->domain . '!'); + exit; + } + + protected function viewBannedInstances() + { + $data = Instance::whereBanned(true) + ->get(['domain', 'user_count', 'status_count']) + ->map(function($d) { + return [ + 'domain' => $d->domain, + 'user_count' => number_format($d->user_count), + 'status_count' => number_format($d->status_count), + ]; + }) + ->toArray(); + table( + ['Domain', 'User Count', 'Status Count'], + $data + ); + } + + protected function viewUnlistedInstances() + { + $data = Instance::whereUnlisted(true) + ->get(['domain', 'user_count', 'status_count', 'banned']) + ->map(function($d) { + return [ + 'domain' => $d->domain, + 'user_count' => number_format($d->user_count), + 'status_count' => number_format($d->status_count), + 'banned' => $d->banned ? '✅' : null + ]; + }) + ->toArray(); + table( + ['Domain', 'User Count', 'Status Count', 'Banned'], + $data + ); + } +} diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index b88401178..a5a4dbb7a 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -70,6 +70,11 @@ class TransformImports extends Command } $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); + if(!$idk) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) { ImportService::clearAttempts($profile->id); diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 8c2726b89..a89b98d03 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -7,56 +7,60 @@ use Illuminate\Foundation\Console\Kernel as ConsoleKernel; class Kernel extends ConsoleKernel { - /** - * The Artisan commands provided by your application. - * - * @var array - */ - protected $commands = [ - // - ]; + /** + * The Artisan commands provided by your application. + * + * @var array + */ + protected $commands = [ + // + ]; - /** - * Define the application's command schedule. - * - * @param \Illuminate\Console\Scheduling\Schedule $schedule - * - * @return void - */ - protected function schedule(Schedule $schedule) - { - $schedule->command('media:optimize')->hourlyAt(40)->onOneServer(); - $schedule->command('media:gc')->hourlyAt(5)->onOneServer(); - $schedule->command('horizon:snapshot')->everyFiveMinutes()->onOneServer(); - $schedule->command('story:gc')->everyFiveMinutes()->onOneServer(); - $schedule->command('gc:failedjobs')->dailyAt(3)->onOneServer(); - $schedule->command('gc:passwordreset')->dailyAt('09:41')->onOneServer(); - $schedule->command('gc:sessions')->twiceDaily(13, 23)->onOneServer(); + /** + * Define the application's command schedule. + * + * @param \Illuminate\Console\Scheduling\Schedule $schedule + * + * @return void + */ + protected function schedule(Schedule $schedule) + { + $schedule->command('media:optimize')->hourlyAt(40)->onOneServer(); + $schedule->command('media:gc')->hourlyAt(5)->onOneServer(); + $schedule->command('horizon:snapshot')->everyFiveMinutes()->onOneServer(); + $schedule->command('story:gc')->everyFiveMinutes()->onOneServer(); + $schedule->command('gc:failedjobs')->dailyAt(3)->onOneServer(); + $schedule->command('gc:passwordreset')->dailyAt('09:41')->onOneServer(); + $schedule->command('gc:sessions')->twiceDaily(13, 23)->onOneServer(); - if (in_array(config_cache('pixelfed.cloud_storage'), ['1', true, 'true']) && config('media.delete_local_after_cloud')) { - $schedule->command('media:s3gc')->hourlyAt(15)->onOneServer(); - } + if(in_array(config_cache('pixelfed.cloud_storage'), ['1', true, 'true']) && config('media.delete_local_after_cloud')) { + $schedule->command('media:s3gc')->hourlyAt(15)->onOneServer(); + } - if (config('import.instagram.enabled')) { - $schedule->command('app:transform-imports')->everyFourMinutes()->onOneServer(); - $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51)->onOneServer(); - $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37)->onOneServer(); - $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32)->onOneServer(); - } + if(config('import.instagram.enabled')) { + $schedule->command('app:transform-imports')->everyTenMinutes()->onOneServer(); + $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51)->onOneServer(); + $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37)->onOneServer(); + $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32)->onOneServer(); - $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21'); - $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25); - } + if(config('import.instagram.storage.cloud.enabled') && (bool) config_cache('pixelfed.cloud_storage')) { + $schedule->command('app:import-upload-media-to-cloud-storage')->hourlyAt(39)->onOneServer(); + } + } + $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer(); + $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer(); + $schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer(); + } - /** - * Register the commands for the application. - * - * @return void - */ - protected function commands() - { - $this->load(__DIR__ . '/Commands'); + /** + * Register the commands for the application. + * + * @return void + */ + protected function commands() + { + $this->load(__DIR__.'/Commands'); - require base_path('routes/console.php'); - } + require base_path('routes/console.php'); + } } diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index b342d68ce..b94a11c92 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -219,6 +219,10 @@ class ApiV1Controller extends Controller if(!$res) { return response()->json(['error' => 'Record not found'], 404); } + if($res && strpos($res['acct'], '@') != -1) { + $domain = parse_url($res['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } return $this->json($res); } @@ -483,6 +487,11 @@ class ApiV1Controller extends Controller $limit = $request->input('limit', 10); $napi = $request->has(self::PF_API_ENTITY_KEY); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($pid) !== intval($account['id'])) { if($account['locked']) { if(!FollowerService::follows($pid, $account['id'])) { @@ -575,6 +584,11 @@ class ApiV1Controller extends Controller $limit = $request->input('limit', 10); $napi = $request->has(self::PF_API_ENTITY_KEY); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($pid) !== intval($account['id'])) { if($account['locked']) { if(!FollowerService::follows($pid, $account['id'])) { @@ -676,6 +690,11 @@ class ApiV1Controller extends Controller return $this->json(['error' => 'Account not found'], 404); } + if($profile && strpos($profile['acct'], '@') != -1) { + $domain = parse_url($profile['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $limit = $request->limit ?? 20; $max_id = $request->max_id; $min_id = $request->min_id; @@ -766,6 +785,11 @@ class ApiV1Controller extends Controller ->whereNull('status') ->findOrFail($id); + if($target && $target->domain) { + $domain = $target->domain; + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $private = (bool) $target->is_private; $remote = (bool) $target->domain; $blocked = UserFilter::whereUserId($target->id) @@ -1252,14 +1276,19 @@ class ApiV1Controller extends Controller $user = $request->user(); abort_if($user->has_roles && !UserRoleService::can('can-like', $user->id), 403, 'Invalid permissions for this action'); - AccountService::setLastActive($user->id); - $status = StatusService::getMastodon($id, false); - abort_unless($status, 400); + abort_unless($status, 404); + + if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) { + $domain = parse_url($status['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } $spid = $status['account']['id']; + AccountService::setLastActive($user->id); + if(intval($spid) !== intval($user->profile_id)) { if($status['visibility'] == 'private') { abort_if(!FollowerService::follows($user->profile_id, $spid), 403); @@ -1404,6 +1433,11 @@ class ApiV1Controller extends Controller return response()->json(['error' => 'Record not found'], 404); } + if($target && strpos($target['acct'], '@') != -1) { + $domain = parse_url($target['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $followRequest = FollowRequest::whereFollowingId($pid)->whereFollowerId($id)->first(); if(!$followRequest) { @@ -2011,6 +2045,11 @@ class ApiV1Controller extends Controller $account = Profile::findOrFail($id); + if($account && $account->domain) { + $domain = $account->domain; + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $count = UserFilterService::muteCount($pid); $maxLimit = intval(config('instance.user_filters.max_user_mutes')); if($count == 0) { @@ -2653,6 +2692,11 @@ class ApiV1Controller extends Controller abort(404); } + if($res && isset($res['account'], $res['account']['acct'], $res['account']['url']) && strpos($res['account']['acct'], '@') != -1) { + $domain = parse_url($res['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $scope = $res['visibility']; if(!in_array($scope, ['public', 'unlisted'])) { if($scope === 'private') { @@ -2697,6 +2741,11 @@ class ApiV1Controller extends Controller return response('', 404); } + if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) { + $domain = parse_url($status['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($status['account']['id']) !== intval($user->profile_id)) { if($status['visibility'] == 'private') { if(!FollowerService::follows($user->profile_id, $status['account']['id'])) { @@ -2780,6 +2829,10 @@ class ApiV1Controller extends Controller $status = Status::findOrFail($id); $account = AccountService::get($status->profile_id, true); abort_if(!$account, 404); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } $author = intval($status->profile_id) === intval($pid) || $user->is_admin; $napi = $request->has(self::PF_API_ENTITY_KEY); @@ -2871,6 +2924,10 @@ class ApiV1Controller extends Controller $pid = $user->profile_id; $status = Status::findOrFail($id); $account = AccountService::get($status->profile_id, true); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } abort_if(!$account, 404); $author = intval($status->profile_id) === intval($pid) || $user->is_admin; $napi = $request->has(self::PF_API_ENTITY_KEY); @@ -3031,7 +3088,7 @@ class ApiV1Controller extends Controller $content = strip_tags($request->input('status')); $rendered = Autolink::create()->autolink($content); - $cw = $user->profile->cw == true ? true : $request->input('sensitive', false); + $cw = $user->profile->cw == true ? true : $request->boolean('sensitive', false); $spoilerText = $cw && $request->filled('spoiler_text') ? $request->input('spoiler_text') : null; if($in_reply_to_id) { @@ -3200,7 +3257,11 @@ class ApiV1Controller extends Controller abort_if($user->has_roles && !UserRoleService::can('can-share', $user->id), 403, 'Invalid permissions for this action'); AccountService::setLastActive($user->id); $status = Status::whereScope('public')->findOrFail($id); - + if($status && ($status->uri || $status->url || $status->object_url)) { + $url = $status->uri ?? $status->url ?? $status->object_url; + $domain = parse_url($url, PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } if(intval($status->profile_id) !== intval($user->profile_id)) { if($status->scope == 'private') { abort_if(!FollowerService::follows($user->profile_id, $status->profile_id), 403); diff --git a/app/Http/Controllers/Api/ApiV2Controller.php b/app/Http/Controllers/Api/ApiV2Controller.php index 93f930cd5..ce15a8a49 100644 --- a/app/Http/Controllers/Api/ApiV2Controller.php +++ b/app/Http/Controllers/Api/ApiV2Controller.php @@ -96,6 +96,9 @@ class ApiV2Controller extends Controller 'streaming' => 'wss://' . config('pixelfed.domain.app'), 'status' => null ], + 'vapid' => [ + 'public_key' => config('webpush.vapid.public_key'), + ], 'accounts' => [ 'max_featured_tags' => 0, ], diff --git a/app/Http/Controllers/PublicApiController.php b/app/Http/Controllers/PublicApiController.php index f888eb512..78008eda4 100644 --- a/app/Http/Controllers/PublicApiController.php +++ b/app/Http/Controllers/PublicApiController.php @@ -42,6 +42,7 @@ use App\Services\{ use App\Jobs\StatusPipeline\NewStatusPipeline; use League\Fractal\Serializer\ArraySerializer; use League\Fractal\Pagination\IlluminatePaginatorAdapter; +use App\Services\InstanceService; class PublicApiController extends Controller { @@ -661,6 +662,10 @@ class PublicApiController extends Controller public function account(Request $request, $id) { $res = AccountService::get($id); + if($res && isset($res['local'], $res['url']) && !$res['local']) { + $domain = parse_url($res['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } return response()->json($res); } @@ -680,6 +685,11 @@ class PublicApiController extends Controller $profile = AccountService::get($id); abort_if(!$profile, 404); + if($profile && isset($profile['local'], $profile['url']) && !$profile['local']) { + $domain = parse_url($profile['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $limit = $request->limit ?? 9; $max_id = $request->max_id; $min_id = $request->min_id; diff --git a/app/Instance.php b/app/Instance.php index 6a7b8e6f2..77752d498 100644 --- a/app/Instance.php +++ b/app/Instance.php @@ -6,63 +6,77 @@ use Illuminate\Database\Eloquent\Model; class Instance extends Model { - protected $fillable = ['domain', 'banned', 'auto_cw', 'unlisted', 'notes']; + protected $casts = [ + 'last_crawled_at' => 'datetime', + 'actors_last_synced_at' => 'datetime', + 'notes' => 'array', + 'nodeinfo_last_fetched' => 'datetime', + 'delivery_next_after' => 'datetime', + ]; - public function profiles() - { - return $this->hasMany(Profile::class, 'domain', 'domain'); - } + protected $fillable = [ + 'domain', + 'banned', + 'auto_cw', + 'unlisted', + 'notes' + ]; - public function statuses() - { - return $this->hasManyThrough( - Status::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function profiles() + { + return $this->hasMany(Profile::class, 'domain', 'domain'); + } - public function reported() - { - return $this->hasManyThrough( - Report::class, - Profile::class, - 'domain', - 'reported_profile_id', - 'domain', - 'id' - ); - } + public function statuses() + { + return $this->hasManyThrough( + Status::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } - public function reports() - { - return $this->hasManyThrough( - Report::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function reported() + { + return $this->hasManyThrough( + Report::class, + Profile::class, + 'domain', + 'reported_profile_id', + 'domain', + 'id' + ); + } - public function media() - { - return $this->hasManyThrough( - Media::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function reports() + { + return $this->hasManyThrough( + Report::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } - public function getUrl() - { - return url("/i/admin/instances/show/{$this->id}"); - } + public function media() + { + return $this->hasManyThrough( + Media::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } + + public function getUrl() + { + return url("/i/admin/instances/show/{$this->id}"); + } } diff --git a/app/Jobs/CommentPipeline/CommentPipeline.php b/app/Jobs/CommentPipeline/CommentPipeline.php index 3b2d896af..1917ecea5 100644 --- a/app/Jobs/CommentPipeline/CommentPipeline.php +++ b/app/Jobs/CommentPipeline/CommentPipeline.php @@ -91,19 +91,21 @@ class CommentPipeline implements ShouldQueue return; } - DB::transaction(function() use($target, $actor, $comment) { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'comment'; - $notification->item_id = $comment->id; - $notification->item_type = "App\Status"; - $notification->save(); + if($target->user_id && $target->domain === null) { + DB::transaction(function() use($target, $actor, $comment) { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'comment'; + $notification->item_id = $comment->id; + $notification->item_type = "App\Status"; + $notification->save(); - NotificationService::setNotification($notification); - NotificationService::set($notification->profile_id, $notification->id); - StatusService::del($comment->id); - }); + NotificationService::setNotification($notification); + NotificationService::set($notification->profile_id, $notification->id); + StatusService::del($comment->id); + }); + } if($exists = Cache::get('status:replies:all:' . $status->id)) { if($exists && $exists->count() == 3) { diff --git a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php index 4969fca2f..77cd5286f 100644 --- a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php +++ b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php @@ -22,9 +22,9 @@ use App\Notification; use App\Services\AccountService; use App\Services\NetworkTimelineService; use App\Services\StatusService; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Cache; +use App\Services\Account\AccountStatService; class DeleteRemoteStatusPipeline implements ShouldQueue { @@ -56,10 +56,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue { $status = $this->status; - if(AccountService::get($status->profile_id, true)) { - DecrementPostCount::dispatch($status->profile_id)->onQueue('low'); - } - + AccountStatService::decrementPostCount($status->profile_id); NetworkTimelineService::del($status->id); StatusService::del($status->id, true); Bookmark::whereStatusId($status->id)->delete(); diff --git a/app/Jobs/FollowPipeline/FollowPipeline.php b/app/Jobs/FollowPipeline/FollowPipeline.php index 225334304..67733919f 100644 --- a/app/Jobs/FollowPipeline/FollowPipeline.php +++ b/app/Jobs/FollowPipeline/FollowPipeline.php @@ -72,16 +72,18 @@ class FollowPipeline implements ShouldQueue $target->save(); AccountService::del($target->id); - try { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'follow'; - $notification->item_id = $target->id; - $notification->item_type = "App\Profile"; - $notification->save(); - } catch (Exception $e) { - Log::error($e); + if($target->user_id && $target->domain === null) { + try { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'follow'; + $notification->item_id = $target->id; + $notification->item_type = "App\Profile"; + $notification->save(); + } catch (Exception $e) { + Log::error($e); + } } } } diff --git a/app/Jobs/ImportPipeline/ImportMediaToCloudPipeline.php b/app/Jobs/ImportPipeline/ImportMediaToCloudPipeline.php new file mode 100644 index 000000000..cdf91e376 --- /dev/null +++ b/app/Jobs/ImportPipeline/ImportMediaToCloudPipeline.php @@ -0,0 +1,129 @@ +importPost->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()]; + } + + /** + * Delete the job if its models no longer exist. + * + * @var bool + */ + public $deleteWhenMissingModels = true; + + /** + * Create a new job instance. + */ + public function __construct(ImportPost $importPost) + { + $this->importPost = $importPost; + } + + /** + * Execute the job. + */ + public function handle(): void + { + $ip = $this->importPost; + + if( + $ip->status_id === null || + $ip->uploaded_to_s3 === true || + (bool) config_cache('pixelfed.cloud_storage') === false) { + return; + } + + $media = Media::whereStatusId($ip->status_id)->get(); + + if(!$media || !$media->count()) { + $importPost = ImportPost::find($ip->id); + $importPost->uploaded_to_s3 = true; + $importPost->save(); + return; + } + + foreach($media as $mediaPart) { + $this->handleMedia($mediaPart); + } + } + + protected function handleMedia($media) + { + $ip = $this->importPost; + + $importPost = ImportPost::find($ip->id); + + if(!$importPost) { + return; + } + + $res = MediaStorageService::move($media); + + $importPost->uploaded_to_s3 = true; + $importPost->save(); + + if(!$res) { + return; + } + + if($res === 'invalid file') { + return; + } + + if($res === 'success') { + if($media->mime === 'video/mp4') { + VideoThumbnailToCloudPipeline::dispatch($media)->onQueue('low'); + } else { + Storage::disk('local')->delete($media->media_path); + } + } + } +} diff --git a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php index b8c79d67f..943281bb4 100644 --- a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php +++ b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php @@ -4,6 +4,7 @@ namespace App\Jobs\InstancePipeline; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -12,45 +13,71 @@ use Illuminate\Support\Facades\Http; use App\Instance; use App\Profile; use App\Services\NodeinfoService; +use Illuminate\Contracts\Cache\Repository; +use Illuminate\Support\Facades\Cache; -class FetchNodeinfoPipeline implements ShouldQueue +class FetchNodeinfoPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $instance; + protected $instance; - /** - * Create a new job instance. - * - * @return void - */ - public function __construct(Instance $instance) - { - $this->instance = $instance; - } + /** + * Create a new job instance. + * + * @return void + */ + public function __construct(Instance $instance) + { + $this->instance = $instance; + } - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - $instance = $this->instance; + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 14400; - $ni = NodeinfoService::get($instance->domain); - if($ni) { - if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) { - $software = $ni['software']['name']; - $instance->software = strtolower(strip_tags($software)); - $instance->last_crawled_at = now(); - $instance->user_count = Profile::whereDomain($instance->domain)->count(); - $instance->save(); - } - } else { - $instance->user_count = Profile::whereDomain($instance->domain)->count(); - $instance->last_crawled_at = now(); - $instance->save(); - } - } + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return $this->instance->id; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $instance = $this->instance; + + if( $instance->nodeinfo_last_fetched && + $instance->nodeinfo_last_fetched->gt(now()->subHours(12)) || + $instance->delivery_timeout && + $instance->delivery_next_after->gt(now()) + ) { + return; + } + + $ni = NodeinfoService::get($instance->domain); + $instance->last_crawled_at = now(); + if($ni) { + if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) { + $software = $ni['software']['name']; + $instance->software = strtolower(strip_tags($software)); + $instance->user_count = Profile::whereDomain($instance->domain)->count(); + $instance->nodeinfo_last_fetched = now(); + $instance->save(); + } + } else { + $instance->delivery_timeout = 1; + $instance->delivery_next_after = now()->addHours(14); + $instance->save(); + } + } } diff --git a/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php b/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php index 477b1f9b3..79df5aa9a 100644 --- a/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php +++ b/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php @@ -61,7 +61,12 @@ class NotificationEpochUpdatePipeline implements ShouldQueue, ShouldBeUniqueUnti */ public function handle(): void { - $rec = Notification::where('created_at', '>', now()->subMonths(6))->first(); + $pid = Cache::get(NotificationService::EPOCH_CACHE_KEY . '6'); + if($pid && $pid > 1) { + $rec = Notification::where('id', '>', $pid)->whereDate('created_at', now()->subMonths(6)->format('Y-m-d'))->first(); + } else { + $rec = Notification::whereDate('created_at', now()->subMonths(6)->format('Y-m-d'))->first(); + } $id = 1; if($rec) { $id = $rec->id; diff --git a/app/Jobs/LikePipeline/LikePipeline.php b/app/Jobs/LikePipeline/LikePipeline.php index b44c90c8b..e55c64f80 100644 --- a/app/Jobs/LikePipeline/LikePipeline.php +++ b/app/Jobs/LikePipeline/LikePipeline.php @@ -79,16 +79,18 @@ class LikePipeline implements ShouldQueue return true; } - try { - $notification = new Notification(); - $notification->profile_id = $status->profile_id; - $notification->actor_id = $actor->id; - $notification->action = 'like'; - $notification->item_id = $status->id; - $notification->item_type = "App\Status"; - $notification->save(); + if($status->uri === null && $status->object_url === null && $status->url === null) { + try { + $notification = new Notification(); + $notification->profile_id = $status->profile_id; + $notification->actor_id = $actor->id; + $notification->action = 'like'; + $notification->item_id = $status->id; + $notification->item_type = "App\Status"; + $notification->save(); - } catch (Exception $e) { + } catch (Exception $e) { + } } } diff --git a/app/Jobs/ProfilePipeline/DecrementPostCount.php b/app/Jobs/ProfilePipeline/DecrementPostCount.php index b463f1dda..74d0523b5 100644 --- a/app/Jobs/ProfilePipeline/DecrementPostCount.php +++ b/app/Jobs/ProfilePipeline/DecrementPostCount.php @@ -35,18 +35,7 @@ class DecrementPostCount implements ShouldQueue */ public function handle() { - $id = $this->id; - - $profile = Profile::find($id); - - if(!$profile) { - return 1; - } - - $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; - $profile->save(); - AccountService::del($id); - - return 1; + // deprecated + return; } } diff --git a/app/Jobs/ProfilePipeline/IncrementPostCount.php b/app/Jobs/ProfilePipeline/IncrementPostCount.php index 1a94f1e6c..a1f9ceca7 100644 --- a/app/Jobs/ProfilePipeline/IncrementPostCount.php +++ b/app/Jobs/ProfilePipeline/IncrementPostCount.php @@ -14,42 +14,12 @@ use App\Profile; use App\Status; use App\Services\AccountService; -class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing +class IncrementPostCount implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $id; - public $timeout = 900; - public $tries = 3; - public $maxExceptions = 1; - public $failOnTimeout = true; - - /** - * The number of seconds after which the job's unique lock will be released. - * - * @var int - */ - public $uniqueFor = 3600; - - /** - * Get the unique ID for the job. - */ - public function uniqueId(): string - { - return 'propipe:ipc:' . $this->id; - } - - /** - * Get the middleware the job should pass through. - * - * @return array - */ - public function middleware(): array - { - return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()]; - } - /** * Create a new job instance. * @@ -67,20 +37,7 @@ class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing */ public function handle() { - $id = $this->id; - - $profile = Profile::find($id); - - if(!$profile) { - return 1; - } - - $profile->status_count = $profile->status_count + 1; - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - AccountService::get($id); - - return 1; + // deprecated + return; } } diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 07a2f6236..a81607755 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -39,8 +39,8 @@ use App\Services\AccountService; use App\Services\CollectionService; use App\Services\StatusService; use App\Jobs\MediaPipeline\MediaDeletePipeline; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Services\NotificationService; +use App\Services\Account\AccountStatService; class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing { @@ -109,9 +109,7 @@ class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing } StatusService::del($status->id, true); - - DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox'); - + AccountStatService::decrementPostCount($status->profile_id); return $this->unlinkRemoveMedia($status); } diff --git a/app/Jobs/StatusPipeline/StatusReplyPipeline.php b/app/Jobs/StatusPipeline/StatusReplyPipeline.php index 35238d293..d8af7b96b 100644 --- a/app/Jobs/StatusPipeline/StatusReplyPipeline.php +++ b/app/Jobs/StatusPipeline/StatusReplyPipeline.php @@ -87,18 +87,20 @@ class StatusReplyPipeline implements ShouldQueue Cache::forget('status:replies:all:' . $reply->id); Cache::forget('status:replies:all:' . $status->id); - DB::transaction(function() use($target, $actor, $status) { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'comment'; - $notification->item_id = $status->id; - $notification->item_type = "App\Status"; - $notification->save(); + if($target->user_id && $target->domain === null) { + DB::transaction(function() use($target, $actor, $status) { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'comment'; + $notification->item_id = $status->id; + $notification->item_type = "App\Status"; + $notification->save(); - NotificationService::setNotification($notification); - NotificationService::set($notification->profile_id, $notification->id); - }); + NotificationService::setNotification($notification); + NotificationService::set($notification->profile_id, $notification->id); + }); + } if($exists = Cache::get('status:replies:all:' . $reply->id)) { if($exists && $exists->count() == 3) { diff --git a/app/Jobs/VideoPipeline/VideoThumbnailToCloudPipeline.php b/app/Jobs/VideoPipeline/VideoThumbnailToCloudPipeline.php new file mode 100644 index 000000000..87931bd7a --- /dev/null +++ b/app/Jobs/VideoPipeline/VideoThumbnailToCloudPipeline.php @@ -0,0 +1,147 @@ +media->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("media:video-thumb-to-cloud:id-{$this->media->id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Media $media) + { + $this->media = $media; + } + + /** + * Execute the job. + */ + public function handle(): void + { + if((bool) config_cache('pixelfed.cloud_storage') === false) { + return; + } + + $media = $this->media; + + if($media->mime != 'video/mp4') { + return; + } + + if($media->profile_id === null || $media->status_id === null) { + return; + } + + if($media->thumbnail_url) { + return; + } + + $base = $media->media_path; + $path = explode('/', $base); + $name = last($path); + + try { + $t = explode('.', $name); + $t = $t[0].'_thumb.jpeg'; + $i = count($path) - 1; + $path[$i] = $t; + $save = implode('/', $path); + $video = FFMpeg::open($base) + ->getFrameFromSeconds(1) + ->export() + ->toDisk('local') + ->save($save); + + if(!$save) { + return; + } + + $media->thumbnail_path = $save; + $p = explode('/', $media->media_path); + array_pop($p); + $pt = explode('/', $save); + $thumbname = array_pop($pt); + $storagePath = implode('/', $p); + $thumb = storage_path('app/' . $save); + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + $media->save(); + + $blurhash = Blurhash::generate($media); + if($blurhash) { + $media->blurhash = $blurhash; + $media->save(); + } + + if(str_starts_with($save, 'public/m/_v2/') && str_ends_with($save, '.jpeg')) { + Storage::delete($save); + } + + if(str_starts_with($media->media_path, 'public/m/_v2/') && str_ends_with($media->media_path, '.mp4')) { + Storage::disk('local')->delete($media->media_path); + } + } catch (Exception $e) { + } + + if($media->status_id) { + Cache::forget('status:transformer:media:attachments:' . $media->status_id); + MediaService::del($media->status_id); + Cache::forget('status:thumb:nsfw0' . $media->status_id); + Cache::forget('status:thumb:nsfw1' . $media->status_id); + Cache::forget('pf:services:sh:id:' . $media->status_id); + StatusService::del($media->status_id); + } + } +} diff --git a/app/Services/Account/AccountStatService.php b/app/Services/Account/AccountStatService.php new file mode 100644 index 000000000..12fd3f94f --- /dev/null +++ b/app/Services/Account/AccountStatService.php @@ -0,0 +1,31 @@ +cloudStore($media); - } + public static function store(Media $media) + { + if(config_cache('pixelfed.cloud_storage') == true) { + (new self())->cloudStore($media); + } - return; - } + return; + } - public static function avatar($avatar, $local = false, $skipRecentCheck = false) - { - return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); - } + public static function move(Media $media) + { + if($media->remote_media) { + return; + } - public static function head($url) - { - $c = new Client(); - try { - $r = $c->request('HEAD', $url); - } catch (RequestException $e) { - return false; - } + if(config_cache('pixelfed.cloud_storage') == true) { + return (new self())->cloudMove($media); + } + return; + } + + public static function avatar($avatar, $local = false, $skipRecentCheck = false) + { + return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); + } + + public static function head($url) + { + $c = new Client(); + try { + $r = $c->request('HEAD', $url); + } catch (RequestException $e) { + return false; + } $h = Arr::mapWithKeys($r->getHeaders(), function($item, $key) { return [strtolower($key) => last($item)]; @@ -55,224 +67,261 @@ class MediaStorageService { $len = (int) $h['content-length']; $mime = $h['content-type']; - if($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) { - return false; - } + if($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) { + return false; + } - return [ - 'length' => $len, - 'mime' => $mime - ]; - } + return [ + 'length' => $len, + 'mime' => $mime + ]; + } - protected function cloudStore($media) - { - if($media->remote_media == true) { - if(config('media.storage.remote.cloud')) { - (new self())->remoteToCloud($media); - } - } else { - (new self())->localToCloud($media); - } - } + protected function cloudStore($media) + { + if($media->remote_media == true) { + if(config('media.storage.remote.cloud')) { + (new self())->remoteToCloud($media); + } + } else { + (new self())->localToCloud($media); + } + } - protected function localToCloud($media) - { - $path = storage_path('app/'.$media->media_path); - $thumb = storage_path('app/'.$media->thumbnail_path); + protected function localToCloud($media) + { + $path = storage_path('app/'.$media->media_path); + $thumb = storage_path('app/'.$media->thumbnail_path); - $p = explode('/', $media->media_path); - $name = array_pop($p); - $pt = explode('/', $media->thumbnail_path); - $thumbname = array_pop($pt); - $storagePath = implode('/', $p); + $p = explode('/', $media->media_path); + $name = array_pop($p); + $pt = explode('/', $media->thumbnail_path); + $thumbname = array_pop($pt); + $storagePath = implode('/', $p); - $url = ResilientMediaStorageService::store($storagePath, $path, $name); - if($thumb) { - $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); - $media->thumbnail_url = $thumbUrl; - } - $media->cdn_url = $url; - $media->optimized_url = $url; - $media->replicated_at = now(); - $media->save(); - if($media->status_id) { - Cache::forget('status:transformer:media:attachments:' . $media->status_id); - MediaService::del($media->status_id); - StatusService::del($media->status_id, false); - } - } + $url = ResilientMediaStorageService::store($storagePath, $path, $name); + if($thumb) { + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + } + $media->cdn_url = $url; + $media->optimized_url = $url; + $media->replicated_at = now(); + $media->save(); + if($media->status_id) { + Cache::forget('status:transformer:media:attachments:' . $media->status_id); + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + } - protected function remoteToCloud($media) - { - $url = $media->remote_url; + protected function remoteToCloud($media) + { + $url = $media->remote_url; - if(!Helpers::validateUrl($url)) { - return; - } + if(!Helpers::validateUrl($url)) { + return; + } - $head = $this->head($media->remote_url); + $head = $this->head($media->remote_url); - if(!$head) { - return; - } + if(!$head) { + return; + } - $mimes = [ - 'image/jpeg', - 'image/png', - 'video/mp4' - ]; + $mimes = [ + 'image/jpeg', + 'image/png', + 'video/mp4' + ]; - $mime = $head['mime']; - $max_size = (int) config_cache('pixelfed.max_photo_size') * 1000; - $media->size = $head['length']; - $media->remote_media = true; - $media->save(); + $mime = $head['mime']; + $max_size = (int) config_cache('pixelfed.max_photo_size') * 1000; + $media->size = $head['length']; + $media->remote_media = true; + $media->save(); - if(!in_array($mime, $mimes)) { - return; - } + if(!in_array($mime, $mimes)) { + return; + } - if($head['length'] >= $max_size) { - return; - } + if($head['length'] >= $max_size) { + return; + } - switch ($mime) { - case 'image/png': - $ext = '.png'; - break; + switch ($mime) { + case 'image/png': + $ext = '.png'; + break; - case 'image/gif': - $ext = '.gif'; - break; + case 'image/gif': + $ext = '.gif'; + break; - case 'image/jpeg': - $ext = '.jpg'; - break; + case 'image/jpeg': + $ext = '.jpg'; + break; - case 'video/mp4': - $ext = '.mp4'; - break; - } + case 'video/mp4': + $ext = '.mp4'; + break; + } - $base = MediaPathService::get($media->profile); - $path = Str::random(40) . $ext; - $tmpBase = storage_path('app/remcache/'); - $tmpPath = $media->profile_id . '-' . $path; - $tmpName = $tmpBase . $tmpPath; - $data = file_get_contents($url, false, null, 0, $head['length']); - file_put_contents($tmpName, $data); - $hash = hash_file('sha256', $tmpName); + $base = MediaPathService::get($media->profile); + $path = Str::random(40) . $ext; + $tmpBase = storage_path('app/remcache/'); + $tmpPath = $media->profile_id . '-' . $path; + $tmpName = $tmpBase . $tmpPath; + $data = file_get_contents($url, false, null, 0, $head['length']); + file_put_contents($tmpName, $data); + $hash = hash_file('sha256', $tmpName); - $disk = Storage::disk(config('filesystems.cloud')); - $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); - $permalink = $disk->url($file); + $disk = Storage::disk(config('filesystems.cloud')); + $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); + $permalink = $disk->url($file); - $media->media_path = $file; - $media->cdn_url = $permalink; - $media->original_sha256 = $hash; - $media->replicated_at = now(); - $media->save(); + $media->media_path = $file; + $media->cdn_url = $permalink; + $media->original_sha256 = $hash; + $media->replicated_at = now(); + $media->save(); - if($media->status_id) { - Cache::forget('status:transformer:media:attachments:' . $media->status_id); - } + if($media->status_id) { + Cache::forget('status:transformer:media:attachments:' . $media->status_id); + } - unlink($tmpName); - } + unlink($tmpName); + } - protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) - { - $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; - $url = $avatar->remote_url; - $driver = $local ? 'local' : config('filesystems.cloud'); + protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) + { + $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; + $url = $avatar->remote_url; + $driver = $local ? 'local' : config('filesystems.cloud'); - if(empty($url) || Helpers::validateUrl($url) == false) { - return; - } + if(empty($url) || Helpers::validateUrl($url) == false) { + return; + } - $head = $this->head($url); + $head = $this->head($url); - if($head == false) { - return; - } + if($head == false) { + return; + } - $mimes = [ - 'application/octet-stream', - 'image/jpeg', - 'image/png', - ]; + $mimes = [ + 'application/octet-stream', + 'image/jpeg', + 'image/png', + ]; - $mime = $head['mime']; - $max_size = (int) config('pixelfed.max_avatar_size') * 1000; + $mime = $head['mime']; + $max_size = (int) config('pixelfed.max_avatar_size') * 1000; - if(!$skipRecentCheck) { - if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { - return; - } - } + if(!$skipRecentCheck) { + if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { + return; + } + } - Cache::forget('avatar:' . $avatar->profile_id); - AccountService::del($avatar->profile_id); + Cache::forget('avatar:' . $avatar->profile_id); + AccountService::del($avatar->profile_id); - // handle pleroma edge case - if(Str::endsWith($mime, '; charset=utf-8')) { - $mime = str_replace('; charset=utf-8', '', $mime); - } + // handle pleroma edge case + if(Str::endsWith($mime, '; charset=utf-8')) { + $mime = str_replace('; charset=utf-8', '', $mime); + } - if(!in_array($mime, $mimes)) { - return; - } + if(!in_array($mime, $mimes)) { + return; + } - if($head['length'] >= $max_size) { - return; - } + if($head['length'] >= $max_size) { + return; + } - $base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id; - $ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png'; - $path = 'avatar_' . strtolower(Str::random(random_int(3,6))) . '.' . $ext; - $tmpBase = storage_path('app/remcache/'); - $tmpPath = 'avatar_' . $avatar->profile_id . '-' . $path; - $tmpName = $tmpBase . $tmpPath; - $data = @file_get_contents($url, false, null, 0, $head['length']); - if(!$data) { - return; - } - file_put_contents($tmpName, $data); + $base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id; + $ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png'; + $path = 'avatar_' . strtolower(Str::random(random_int(3,6))) . '.' . $ext; + $tmpBase = storage_path('app/remcache/'); + $tmpPath = 'avatar_' . $avatar->profile_id . '-' . $path; + $tmpName = $tmpBase . $tmpPath; + $data = @file_get_contents($url, false, null, 0, $head['length']); + if(!$data) { + return; + } + file_put_contents($tmpName, $data); - $mimeCheck = Storage::mimeType('remcache/' . $tmpPath); + $mimeCheck = Storage::mimeType('remcache/' . $tmpPath); - if(!$mimeCheck || !in_array($mimeCheck, ['image/png', 'image/jpeg'])) { - $avatar->last_fetched_at = now(); - $avatar->save(); - unlink($tmpName); - return; - } + if(!$mimeCheck || !in_array($mimeCheck, ['image/png', 'image/jpeg'])) { + $avatar->last_fetched_at = now(); + $avatar->save(); + unlink($tmpName); + return; + } - $disk = Storage::disk($driver); - $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); - $permalink = $disk->url($file); + $disk = Storage::disk($driver); + $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); + $permalink = $disk->url($file); - $avatar->media_path = $base . '/' . $path; - $avatar->is_remote = true; - $avatar->cdn_url = $local ? config('app.url') . $permalink : $permalink; - $avatar->size = $head['length']; - $avatar->change_count = $avatar->change_count + 1; - $avatar->last_fetched_at = now(); - $avatar->save(); + $avatar->media_path = $base . '/' . $path; + $avatar->is_remote = true; + $avatar->cdn_url = $local ? config('app.url') . $permalink : $permalink; + $avatar->size = $head['length']; + $avatar->change_count = $avatar->change_count + 1; + $avatar->last_fetched_at = now(); + $avatar->save(); - Cache::forget('avatar:' . $avatar->profile_id); - AccountService::del($avatar->profile_id); - AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); + Cache::forget('avatar:' . $avatar->profile_id); + AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); - unlink($tmpName); - } + unlink($tmpName); + } - public static function delete(Media $media, $confirm = false) - { - if(!$confirm) { - return; - } - MediaDeletePipeline::dispatch($media)->onQueue('mmo'); - } + public static function delete(Media $media, $confirm = false) + { + if(!$confirm) { + return; + } + MediaDeletePipeline::dispatch($media)->onQueue('mmo'); + } + + protected function cloudMove($media) + { + if(!Storage::exists($media->media_path)) { + return 'invalid file'; + } + + $path = storage_path('app/'.$media->media_path); + $thumb = false; + if($media->thumbnail_path) { + $thumb = storage_path('app/'.$media->thumbnail_path); + $pt = explode('/', $media->thumbnail_path); + $thumbname = array_pop($pt); + } + + $p = explode('/', $media->media_path); + $name = array_pop($p); + $storagePath = implode('/', $p); + + $url = ResilientMediaStorageService::store($storagePath, $path, $name); + if($thumb) { + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + } + $media->cdn_url = $url; + $media->optimized_url = $url; + $media->replicated_at = now(); + $media->save(); + + if($media->status_id) { + Cache::forget('status:transformer:media:attachments:' . $media->status_id); + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'success'; + } } diff --git a/app/Services/NodeinfoService.php b/app/Services/NodeinfoService.php index 10575ff9f..6284538f0 100644 --- a/app/Services/NodeinfoService.php +++ b/app/Services/NodeinfoService.php @@ -22,7 +22,10 @@ class NodeinfoService $wk = $url . '/.well-known/nodeinfo'; try { - $res = Http::withHeaders($headers) + $res = Http::withOptions([ + 'allow_redirects' => false, + ]) + ->withHeaders($headers) ->timeout(5) ->get($wk); } catch (RequestException $e) { @@ -61,7 +64,10 @@ class NodeinfoService } try { - $res = Http::withHeaders($headers) + $res = Http::withOptions([ + 'allow_redirects' => false, + ]) + ->withHeaders($headers) ->timeout(5) ->get($href); } catch (RequestException $e) { diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 612c38d75..5819dc0bc 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -39,10 +39,9 @@ use App\Jobs\HomeFeedPipeline\FeedInsertRemotePipeline; use App\Util\Media\License; use App\Models\Poll; use Illuminate\Contracts\Cache\LockTimeoutException; -use App\Jobs\ProfilePipeline\IncrementPostCount; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Services\DomainService; use App\Services\UserFilterService; +use App\Services\Account\AccountStatService; class Helpers { @@ -536,7 +535,7 @@ class Helpers { } } - IncrementPostCount::dispatch($pid)->onQueue('low'); + AccountStatService::incrementPostCount($pid); if( $status->in_reply_to_id === null && in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) @@ -549,10 +548,11 @@ class Helpers { public static function getSensitive($activity, $url) { - $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); - $url = isset($activity['url']) ? self::pluckval($activity['url']) : $id; - $urlDomain = parse_url($url, PHP_URL_HOST); + if(!$url || !strlen($url)) { + return true; + } + $urlDomain = parse_url($url, PHP_URL_HOST); $cw = isset($activity['sensitive']) ? (bool) $activity['sensitive'] : false; if(in_array($urlDomain, InstanceService::getNsfwDomains())) { diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index e26f0a48c..5c9959e17 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -48,8 +48,6 @@ use App\Services\UserFilterService; use App\Services\NetworkTimelineService; use App\Models\Conversation; use App\Models\RemoteReport; -use App\Jobs\ProfilePipeline\IncrementPostCount; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline; class Inbox diff --git a/config/horizon.php b/config/horizon.php index f9cfd960e..5aa37f2fe 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -2,201 +2,202 @@ return [ - /* - |-------------------------------------------------------------------------- - | Horizon Domain - |-------------------------------------------------------------------------- - | - | This is the subdomain where Horizon will be accessible from. If this - | setting is null, Horizon will reside under the same domain as the - | application. Otherwise, this value will serve as the subdomain. - | - */ + /* + |-------------------------------------------------------------------------- + | Horizon Domain + |-------------------------------------------------------------------------- + | + | This is the subdomain where Horizon will be accessible from. If this + | setting is null, Horizon will reside under the same domain as the + | application. Otherwise, this value will serve as the subdomain. + | + */ - 'domain' => null, + 'domain' => null, - /* - |-------------------------------------------------------------------------- - | Horizon Path - |-------------------------------------------------------------------------- - | - | This is the URI path where Horizon will be accessible from. Feel free - | to change this path to anything you like. Note that the URI will not - | affect the paths of its internal API that aren't exposed to users. - | - */ + /* + |-------------------------------------------------------------------------- + | Horizon Path + |-------------------------------------------------------------------------- + | + | This is the URI path where Horizon will be accessible from. Feel free + | to change this path to anything you like. Note that the URI will not + | affect the paths of its internal API that aren't exposed to users. + | + */ - 'path' => 'horizon', + 'path' => 'horizon', - /* - |-------------------------------------------------------------------------- - | Horizon Redis Connection - |-------------------------------------------------------------------------- - | - | This is the name of the Redis connection where Horizon will store the - | meta information required for it to function. It includes the list - | of supervisors, failed jobs, job metrics, and other information. - | - */ + /* + |-------------------------------------------------------------------------- + | Horizon Redis Connection + |-------------------------------------------------------------------------- + | + | This is the name of the Redis connection where Horizon will store the + | meta information required for it to function. It includes the list + | of supervisors, failed jobs, job metrics, and other information. + | + */ - 'use' => 'default', + 'use' => 'default', - /* - |-------------------------------------------------------------------------- - | Horizon Redis Prefix - |-------------------------------------------------------------------------- - | - | This prefix will be used when storing all Horizon data in Redis. You - | may modify the prefix when you are running multiple installations - | of Horizon on the same server so that they don't have problems. - | - */ + /* + |-------------------------------------------------------------------------- + | Horizon Redis Prefix + |-------------------------------------------------------------------------- + | + | This prefix will be used when storing all Horizon data in Redis. You + | may modify the prefix when you are running multiple installations + | of Horizon on the same server so that they don't have problems. + | + */ - 'prefix' => env('HORIZON_PREFIX', 'horizon-'), + 'prefix' => env('HORIZON_PREFIX', 'horizon-'), - /* - |-------------------------------------------------------------------------- - | Horizon Route Middleware - |-------------------------------------------------------------------------- - | - | These middleware will get attached onto each Horizon route, giving you - | the chance to add your own middleware to this list or change any of - | the existing middleware. Or, you can simply stick with this list. - | - */ + /* + |-------------------------------------------------------------------------- + | Horizon Route Middleware + |-------------------------------------------------------------------------- + | + | These middleware will get attached onto each Horizon route, giving you + | the chance to add your own middleware to this list or change any of + | the existing middleware. Or, you can simply stick with this list. + | + */ - 'middleware' => ['web'], + 'middleware' => ['web'], - /* - |-------------------------------------------------------------------------- - | Queue Wait Time Thresholds - |-------------------------------------------------------------------------- - | - | This option allows you to configure when the LongWaitDetected event - | will be fired. Every connection / queue combination may have its - | own, unique threshold (in seconds) before this event is fired. - | - */ + /* + |-------------------------------------------------------------------------- + | Queue Wait Time Thresholds + |-------------------------------------------------------------------------- + | + | This option allows you to configure when the LongWaitDetected event + | will be fired. Every connection / queue combination may have its + | own, unique threshold (in seconds) before this event is fired. + | + */ - 'waits' => [ - 'redis:feed' => 30, - 'redis:follow' => 30, - 'redis:shared' => 30, - 'redis:default' => 30, - 'redis:inbox' => 30, - 'redis:low' => 30, - 'redis:high' => 30, - 'redis:delete' => 30, - 'redis:story' => 30, - 'redis:mmo' => 30, - ], + 'waits' => [ + 'redis:feed' => 30, + 'redis:follow' => 30, + 'redis:shared' => 30, + 'redis:default' => 30, + 'redis:inbox' => 30, + 'redis:low' => 30, + 'redis:high' => 30, + 'redis:delete' => 30, + 'redis:story' => 30, + 'redis:mmo' => 30, + 'redis:intbg' => 30, + ], - /* - |-------------------------------------------------------------------------- - | Job Trimming Times - |-------------------------------------------------------------------------- - | - | Here you can configure for how long (in minutes) you desire Horizon to - | persist the recent and failed jobs. Typically, recent jobs are kept - | for one hour while all failed jobs are stored for an entire week. - | - */ + /* + |-------------------------------------------------------------------------- + | Job Trimming Times + |-------------------------------------------------------------------------- + | + | Here you can configure for how long (in minutes) you desire Horizon to + | persist the recent and failed jobs. Typically, recent jobs are kept + | for one hour while all failed jobs are stored for an entire week. + | + */ - 'trim' => [ - 'recent' => 60, - 'pending' => 60, - 'completed' => 60, - 'recent_failed' => 10080, - 'failed' => 10080, - 'monitored' => 10080, - ], + 'trim' => [ + 'recent' => 60, + 'pending' => 60, + 'completed' => 60, + 'recent_failed' => 10080, + 'failed' => 10080, + 'monitored' => 10080, + ], - /* - |-------------------------------------------------------------------------- - | Metrics - |-------------------------------------------------------------------------- - | - | Here you can configure how many snapshots should be kept to display in - | the metrics graph. This will get used in combination with Horizon's - | `horizon:snapshot` schedule to define how long to retain metrics. - | - */ + /* + |-------------------------------------------------------------------------- + | Metrics + |-------------------------------------------------------------------------- + | + | Here you can configure how many snapshots should be kept to display in + | the metrics graph. This will get used in combination with Horizon's + | `horizon:snapshot` schedule to define how long to retain metrics. + | + */ - 'metrics' => [ - 'trim_snapshots' => [ - 'job' => 24, - 'queue' => 24, - ], - ], + 'metrics' => [ + 'trim_snapshots' => [ + 'job' => 24, + 'queue' => 24, + ], + ], - /* - |-------------------------------------------------------------------------- - | Fast Termination - |-------------------------------------------------------------------------- - | - | When this option is enabled, Horizon's "terminate" command will not - | wait on all of the workers to terminate unless the --wait option - | is provided. Fast termination can shorten deployment delay by - | allowing a new instance of Horizon to start while the last - | instance will continue to terminate each of its workers. - | - */ + /* + |-------------------------------------------------------------------------- + | Fast Termination + |-------------------------------------------------------------------------- + | + | When this option is enabled, Horizon's "terminate" command will not + | wait on all of the workers to terminate unless the --wait option + | is provided. Fast termination can shorten deployment delay by + | allowing a new instance of Horizon to start while the last + | instance will continue to terminate each of its workers. + | + */ - 'fast_termination' => false, + 'fast_termination' => false, - /* - |-------------------------------------------------------------------------- - | Memory Limit (MB) - |-------------------------------------------------------------------------- - | - | This value describes the maximum amount of memory the Horizon worker - | may consume before it is terminated and restarted. You should set - | this value according to the resources available to your server. - | - */ + /* + |-------------------------------------------------------------------------- + | Memory Limit (MB) + |-------------------------------------------------------------------------- + | + | This value describes the maximum amount of memory the Horizon worker + | may consume before it is terminated and restarted. You should set + | this value according to the resources available to your server. + | + */ - 'memory_limit' => env('HORIZON_MEMORY_LIMIT', 64), + 'memory_limit' => env('HORIZON_MEMORY_LIMIT', 64), - /* - |-------------------------------------------------------------------------- - | Queue Worker Configuration - |-------------------------------------------------------------------------- - | - | Here you may define the queue worker settings used by your application - | in all environments. These supervisors and settings handle all your - | queued jobs and will be provisioned by Horizon during deployment. - | - */ + /* + |-------------------------------------------------------------------------- + | Queue Worker Configuration + |-------------------------------------------------------------------------- + | + | Here you may define the queue worker settings used by your application + | in all environments. These supervisors and settings handle all your + | queued jobs and will be provisioned by Horizon during deployment. + | + */ - 'environments' => [ - 'production' => [ - 'supervisor-1' => [ - 'connection' => 'redis', - 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'], - 'balance' => env('HORIZON_BALANCE_STRATEGY', 'auto'), - 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), - 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 20), - 'memory' => env('HORIZON_SUPERVISOR_MEMORY', 64), - 'tries' => env('HORIZON_SUPERVISOR_TRIES', 3), - 'nice' => env('HORIZON_SUPERVISOR_NICE', 0), - 'timeout' => env('HORIZON_SUPERVISOR_TIMEOUT', 300), - ], - ], + 'environments' => [ + 'production' => [ + 'supervisor-1' => [ + 'connection' => 'redis', + 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'], + 'balance' => env('HORIZON_BALANCE_STRATEGY', 'auto'), + 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), + 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 20), + 'memory' => env('HORIZON_SUPERVISOR_MEMORY', 64), + 'tries' => env('HORIZON_SUPERVISOR_TRIES', 3), + 'nice' => env('HORIZON_SUPERVISOR_NICE', 0), + 'timeout' => env('HORIZON_SUPERVISOR_TIMEOUT', 300), + ], + ], - 'local' => [ - 'supervisor-1' => [ - 'connection' => 'redis', - 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'], - 'balance' => 'auto', - 'minProcesses' => 1, - 'maxProcesses' => 20, - 'memory' => 128, - 'tries' => 3, - 'nice' => 0, - 'timeout' => 300 - ], - ], - ], + 'local' => [ + 'supervisor-1' => [ + 'connection' => 'redis', + 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'], + 'balance' => 'auto', + 'minProcesses' => 1, + 'maxProcesses' => 20, + 'memory' => 128, + 'tries' => 3, + 'nice' => 0, + 'timeout' => 300 + ], + ], + ], - 'darkmode' => env('HORIZON_DARKMODE', false), + 'darkmode' => env('HORIZON_DARKMODE', false), ]; diff --git a/config/import.php b/config/import.php index 2d1af28e1..f754da490 100644 --- a/config/import.php +++ b/config/import.php @@ -39,6 +39,12 @@ return [ // Limit to specific user ids, in comma separated format 'user_ids' => env('PF_IMPORT_IG_PERM_ONLY_USER_IDS', null), + ], + + 'storage' => [ + 'cloud' => [ + 'enabled' => env('PF_IMPORT_IG_CLOUD_STORAGE', env('PF_ENABLE_CLOUD', false)), + ] ] ] ]; diff --git a/database/migrations/2018_12_22_055940_add_account_status_to_profiles_table.php b/database/migrations/2018_12_22_055940_add_account_status_to_profiles_table.php index 04a88060e..097e86753 100644 --- a/database/migrations/2018_12_22_055940_add_account_status_to_profiles_table.php +++ b/database/migrations/2018_12_22_055940_add_account_status_to_profiles_table.php @@ -54,12 +54,14 @@ class AddAccountStatusToProfilesTable extends Migration $table->string('hub_url')->nullable(); }); - Schema::table('stories', function (Blueprint $table) { - $table->dropColumn('id'); - }); - Schema::table('stories', function (Blueprint $table) { - $table->bigIncrements('bigIncrements')->first(); - }); + if (Schema::hasTable('stories')) { + Schema::table('stories', function (Blueprint $table) { + $table->dropColumn('id'); + }); + Schema::table('stories', function (Blueprint $table) { + $table->bigIncrements('bigIncrements')->first(); + }); + } Schema::table('profiles', function (Blueprint $table) { $table->dropColumn('status'); diff --git a/database/migrations/2019_01_12_054413_stories.php b/database/migrations/2019_01_12_054413_stories.php index a61c447de..f58a8cf38 100644 --- a/database/migrations/2019_01_12_054413_stories.php +++ b/database/migrations/2019_01_12_054413_stories.php @@ -60,13 +60,7 @@ class Stories extends Migration { Schema::dropIfExists('story_items'); Schema::dropIfExists('story_views'); - - Schema::table('stories', function (Blueprint $table) { - $table->dropColumn(['title','preview_photo','local_only','is_live','broadcast_url','broadcast_key']); - }); - - Schema::table('story_reactions', function (Blueprint $table) { - $table->dropColumn('story_id'); - }); + Schema::dropIfExists('story_reactions'); + Schema::dropIfExists('stories'); } } diff --git a/database/migrations/2019_12_10_023604_create_newsroom_table.php b/database/migrations/2019_12_10_023604_create_newsroom_table.php index 2651d5c4d..b463f5624 100644 --- a/database/migrations/2019_12_10_023604_create_newsroom_table.php +++ b/database/migrations/2019_12_10_023604_create_newsroom_table.php @@ -40,6 +40,6 @@ class CreateNewsroomTable extends Migration */ public function down() { - Schema::dropIfExists('site_news'); + Schema::dropIfExists('newsroom'); } } diff --git a/database/migrations/2021_01_14_034521_add_cache_locks_table.php b/database/migrations/2021_01_14_034521_add_cache_locks_table.php index 121c69a37..07889b490 100644 --- a/database/migrations/2021_01_14_034521_add_cache_locks_table.php +++ b/database/migrations/2021_01_14_034521_add_cache_locks_table.php @@ -27,6 +27,6 @@ class AddCacheLocksTable extends Migration */ public function down() { - Schema::dropTable('cache_locks'); + Schema::dropIfExists('cache_locks'); } } diff --git a/database/migrations/2021_07_23_062326_add_compose_settings_to_user_settings_table.php b/database/migrations/2021_07_23_062326_add_compose_settings_to_user_settings_table.php index 58837cab3..49a9b2c58 100644 --- a/database/migrations/2021_07_23_062326_add_compose_settings_to_user_settings_table.php +++ b/database/migrations/2021_07_23_062326_add_compose_settings_to_user_settings_table.php @@ -33,14 +33,25 @@ class AddComposeSettingsToUserSettingsTable extends Migration public function down() { Schema::table('user_settings', function (Blueprint $table) { - $table->dropColumn('compose_settings'); + if (Schema::hasColumn('user_settings', 'compose_settings')) { + $table->dropColumn('compose_settings'); + } }); Schema::table('media', function (Blueprint $table) { $table->string('caption')->change(); - $table->dropIndex('profile_id'); - $table->dropIndex('mime'); - $table->dropIndex('license'); + + $schemaManager = Schema::getConnection()->getDoctrineSchemaManager(); + $indexesFound = $schemaManager->listTableIndexes('media'); + if (array_key_exists('media_profile_id_index', $indexesFound)) { + $table->dropIndex('media_profile_id_index'); + } + if (array_key_exists('media_mime_index', $indexesFound)) { + $table->dropIndex('media_mime_index'); + } + if (array_key_exists('media_license_index', $indexesFound)) { + $table->dropIndex('media_license_index'); + } }); } } diff --git a/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php b/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php new file mode 100644 index 000000000..d6e413768 --- /dev/null +++ b/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php @@ -0,0 +1,36 @@ +boolean('active_deliver')->nullable()->index()->after('domain'); + $table->boolean('valid_nodeinfo')->nullable(); + $table->timestamp('nodeinfo_last_fetched')->nullable(); + $table->boolean('delivery_timeout')->default(false); + $table->timestamp('delivery_next_after')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('instances', function (Blueprint $table) { + $table->dropColumn('active_deliver'); + $table->dropColumn('valid_nodeinfo'); + $table->dropColumn('nodeinfo_last_fetched'); + $table->dropColumn('delivery_timeout'); + $table->dropColumn('delivery_next_after'); + }); + } +}; diff --git a/database/migrations/2024_01_09_052419_create_parental_controls_table.php b/database/migrations/2024_01_09_052419_create_parental_controls_table.php index bf803e4c0..6713e6849 100644 --- a/database/migrations/2024_01_09_052419_create_parental_controls_table.php +++ b/database/migrations/2024_01_09_052419_create_parental_controls_table.php @@ -28,7 +28,7 @@ return new class extends Migration $schemaManager = Schema::getConnection()->getDoctrineSchemaManager(); $indexesFound = $schemaManager->listTableIndexes('user_roles'); if (array_key_exists('user_roles_profile_id_unique', $indexesFound)) { - $table->dropIndex('user_roles_profile_id_unique'); + $table->dropUnique('user_roles_profile_id_unique'); } $table->unsignedBigInteger('profile_id')->unique()->nullable()->index()->change(); }); @@ -42,7 +42,11 @@ return new class extends Migration Schema::dropIfExists('parental_controls'); Schema::table('user_roles', function (Blueprint $table) { - $table->dropIndex('user_roles_profile_id_unique'); + $schemaManager = Schema::getConnection()->getDoctrineSchemaManager(); + $indexesFound = $schemaManager->listTableIndexes('user_roles'); + if (array_key_exists('user_roles_profile_id_unique', $indexesFound)) { + $table->dropUnique('user_roles_profile_id_unique'); + } $table->unsignedBigInteger('profile_id')->unique()->index()->change(); }); } diff --git a/public/js/account-import.js b/public/js/account-import.js index 74d4d9e4d..b68bfb495 100644 Binary files a/public/js/account-import.js and b/public/js/account-import.js differ diff --git a/public/js/admin.js b/public/js/admin.js index d71cbfa94..a216de097 100644 Binary files a/public/js/admin.js and b/public/js/admin.js differ diff --git a/public/js/collectioncompose.js b/public/js/collectioncompose.js index e295e7965..d729c9132 100644 Binary files a/public/js/collectioncompose.js and b/public/js/collectioncompose.js differ diff --git a/public/js/collections.js b/public/js/collections.js index f59c88863..bce9e53b6 100644 Binary files a/public/js/collections.js and b/public/js/collections.js differ diff --git a/public/js/compose.chunk.10e7f993dcc726f9.js b/public/js/compose.chunk.10e7f993dcc726f9.js deleted file mode 100644 index 106a118f4..000000000 Binary files a/public/js/compose.chunk.10e7f993dcc726f9.js and /dev/null differ diff --git a/public/js/compose.chunk.1ac292c93b524406.js b/public/js/compose.chunk.1ac292c93b524406.js new file mode 100644 index 000000000..ed1b93071 Binary files /dev/null and b/public/js/compose.chunk.1ac292c93b524406.js differ diff --git a/public/js/compose.js b/public/js/compose.js index 699a19dad..e64303401 100644 Binary files a/public/js/compose.js and b/public/js/compose.js differ diff --git a/public/js/daci.chunk.b17a0b11877389d7.js b/public/js/daci.chunk.8d4acc1db3f27a51.js similarity index 67% rename from public/js/daci.chunk.b17a0b11877389d7.js rename to public/js/daci.chunk.8d4acc1db3f27a51.js index ccbc67f41..75059480a 100644 Binary files a/public/js/daci.chunk.b17a0b11877389d7.js and b/public/js/daci.chunk.8d4acc1db3f27a51.js differ diff --git a/public/js/discover.chunk.9606885dad3c8a99.js b/public/js/discover.chunk.b1846efb6bd1e43c.js similarity index 61% rename from public/js/discover.chunk.9606885dad3c8a99.js rename to public/js/discover.chunk.b1846efb6bd1e43c.js index 7892b83f2..fb2ff1482 100644 Binary files a/public/js/discover.chunk.9606885dad3c8a99.js and b/public/js/discover.chunk.b1846efb6bd1e43c.js differ diff --git a/public/js/discover~findfriends.chunk.02be60ab26503531.js b/public/js/discover~findfriends.chunk.941b524eee8b8d63.js similarity index 67% rename from public/js/discover~findfriends.chunk.02be60ab26503531.js rename to public/js/discover~findfriends.chunk.941b524eee8b8d63.js index 60d93a0a6..cbf5bdb24 100644 Binary files a/public/js/discover~findfriends.chunk.02be60ab26503531.js and b/public/js/discover~findfriends.chunk.941b524eee8b8d63.js differ diff --git a/public/js/discover~hashtag.bundle.6c2ff384b17ea58d.js b/public/js/discover~hashtag.bundle.6c2ff384b17ea58d.js new file mode 100644 index 000000000..3ef77ac5d Binary files /dev/null and b/public/js/discover~hashtag.bundle.6c2ff384b17ea58d.js differ diff --git a/public/js/discover~hashtag.bundle.9cfffc517f35044e.js b/public/js/discover~hashtag.bundle.9cfffc517f35044e.js deleted file mode 100644 index afb2b3de0..000000000 Binary files a/public/js/discover~hashtag.bundle.9cfffc517f35044e.js and /dev/null differ diff --git a/public/js/discover~memories.chunk.ce9cc6446020e9b3.js b/public/js/discover~memories.chunk.7d917826c3e9f17b.js similarity index 67% rename from public/js/discover~memories.chunk.ce9cc6446020e9b3.js rename to public/js/discover~memories.chunk.7d917826c3e9f17b.js index ec86c87f7..f8ed91112 100644 Binary files a/public/js/discover~memories.chunk.ce9cc6446020e9b3.js and b/public/js/discover~memories.chunk.7d917826c3e9f17b.js differ diff --git a/public/js/discover~myhashtags.chunk.6eab2414b2b16e19.js b/public/js/discover~myhashtags.chunk.a72fc4882db8afd3.js similarity index 73% rename from public/js/discover~myhashtags.chunk.6eab2414b2b16e19.js rename to public/js/discover~myhashtags.chunk.a72fc4882db8afd3.js index 31039621f..adab1bc44 100644 Binary files a/public/js/discover~myhashtags.chunk.6eab2414b2b16e19.js and b/public/js/discover~myhashtags.chunk.a72fc4882db8afd3.js differ diff --git a/public/js/discover~serverfeed.chunk.0f2dcc473fdce17e.js b/public/js/discover~serverfeed.chunk.8365948d1867de3a.js similarity index 67% rename from public/js/discover~serverfeed.chunk.0f2dcc473fdce17e.js rename to public/js/discover~serverfeed.chunk.8365948d1867de3a.js index ed30efc2f..fba06dbca 100644 Binary files a/public/js/discover~serverfeed.chunk.0f2dcc473fdce17e.js and b/public/js/discover~serverfeed.chunk.8365948d1867de3a.js differ diff --git a/public/js/discover~settings.chunk.732c1f76a00d9204.js b/public/js/discover~settings.chunk.be88dc5ba1a24a7d.js similarity index 68% rename from public/js/discover~settings.chunk.732c1f76a00d9204.js rename to public/js/discover~settings.chunk.be88dc5ba1a24a7d.js index 959ab4d3f..76bbfcf7d 100644 Binary files a/public/js/discover~settings.chunk.732c1f76a00d9204.js and b/public/js/discover~settings.chunk.be88dc5ba1a24a7d.js differ diff --git a/public/js/dms~message.chunk.15157ff4a6c17cc7.js b/public/js/dms~message.chunk.76edeafda3d92320.js similarity index 63% rename from public/js/dms~message.chunk.15157ff4a6c17cc7.js rename to public/js/dms~message.chunk.76edeafda3d92320.js index e74ff4325..17a5262d9 100644 Binary files a/public/js/dms~message.chunk.15157ff4a6c17cc7.js and b/public/js/dms~message.chunk.76edeafda3d92320.js differ diff --git a/public/js/home.chunk.351f55e9d09b6482.js b/public/js/home.chunk.351f55e9d09b6482.js deleted file mode 100644 index c6f3e4d4f..000000000 Binary files a/public/js/home.chunk.351f55e9d09b6482.js and /dev/null differ diff --git a/public/js/home.chunk.f3f4f632025b560f.js b/public/js/home.chunk.f3f4f632025b560f.js new file mode 100644 index 000000000..76c5f028a Binary files /dev/null and b/public/js/home.chunk.f3f4f632025b560f.js differ diff --git a/public/js/home.chunk.351f55e9d09b6482.js.LICENSE.txt b/public/js/home.chunk.f3f4f632025b560f.js.LICENSE.txt similarity index 100% rename from public/js/home.chunk.351f55e9d09b6482.js.LICENSE.txt rename to public/js/home.chunk.f3f4f632025b560f.js.LICENSE.txt diff --git a/public/js/landing.js b/public/js/landing.js index 8a3b2322c..c9da7051d 100644 Binary files a/public/js/landing.js and b/public/js/landing.js differ diff --git a/public/js/manifest.js b/public/js/manifest.js index d6fe923ca..5f337f9b3 100644 Binary files a/public/js/manifest.js and b/public/js/manifest.js differ diff --git a/public/js/portfolio.js b/public/js/portfolio.js index ce1e7097f..f28a7af06 100644 Binary files a/public/js/portfolio.js and b/public/js/portfolio.js differ diff --git a/public/js/post.chunk.23fc9e82d4fadc83.js b/public/js/post.chunk.eb9804ff282909ae.js similarity index 72% rename from public/js/post.chunk.23fc9e82d4fadc83.js rename to public/js/post.chunk.eb9804ff282909ae.js index 6e8aa73f4..130499b2c 100644 Binary files a/public/js/post.chunk.23fc9e82d4fadc83.js and b/public/js/post.chunk.eb9804ff282909ae.js differ diff --git a/public/js/post.chunk.23fc9e82d4fadc83.js.LICENSE.txt b/public/js/post.chunk.eb9804ff282909ae.js.LICENSE.txt similarity index 100% rename from public/js/post.chunk.23fc9e82d4fadc83.js.LICENSE.txt rename to public/js/post.chunk.eb9804ff282909ae.js.LICENSE.txt diff --git a/public/js/profile.chunk.0e5bd852054d6355.js b/public/js/profile.chunk.d52916cb68c9a146.js similarity index 64% rename from public/js/profile.chunk.0e5bd852054d6355.js rename to public/js/profile.chunk.d52916cb68c9a146.js index acf4175dc..f6ea10fa0 100644 Binary files a/public/js/profile.chunk.0e5bd852054d6355.js and b/public/js/profile.chunk.d52916cb68c9a146.js differ diff --git a/public/js/profile.js b/public/js/profile.js index a9fcd40df..0c28afe41 100644 Binary files a/public/js/profile.js and b/public/js/profile.js differ diff --git a/public/js/profile~followers.bundle.731f680cfb96563d.js b/public/js/profile~followers.bundle.5deed93248f20662.js similarity index 70% rename from public/js/profile~followers.bundle.731f680cfb96563d.js rename to public/js/profile~followers.bundle.5deed93248f20662.js index 51805c382..171ddf72d 100644 Binary files a/public/js/profile~followers.bundle.731f680cfb96563d.js and b/public/js/profile~followers.bundle.5deed93248f20662.js differ diff --git a/public/js/profile~following.bundle.3d95796c9f1678dd.js b/public/js/profile~following.bundle.d2b3b1fc2e05dbd3.js similarity index 70% rename from public/js/profile~following.bundle.3d95796c9f1678dd.js rename to public/js/profile~following.bundle.d2b3b1fc2e05dbd3.js index 24db7dafa..9a35b3e2b 100644 Binary files a/public/js/profile~following.bundle.3d95796c9f1678dd.js and b/public/js/profile~following.bundle.d2b3b1fc2e05dbd3.js differ diff --git a/public/js/spa.js b/public/js/spa.js index b496ba432..baee816fd 100644 Binary files a/public/js/spa.js and b/public/js/spa.js differ diff --git a/public/js/status.js b/public/js/status.js index fb8eb6b7e..0bced46dd 100644 Binary files a/public/js/status.js and b/public/js/status.js differ diff --git a/public/js/stories.js b/public/js/stories.js index e38874d8e..545526829 100644 Binary files a/public/js/stories.js and b/public/js/stories.js differ diff --git a/public/js/timeline.js b/public/js/timeline.js index a0ec87563..ed0fc1214 100644 Binary files a/public/js/timeline.js and b/public/js/timeline.js differ diff --git a/public/mix-manifest.json b/public/mix-manifest.json index 29b3788c0..68c9c44bd 100644 Binary files a/public/mix-manifest.json and b/public/mix-manifest.json differ diff --git a/resources/assets/components/AccountImport.vue b/resources/assets/components/AccountImport.vue index 6e1fe9fcf..f7b7279f5 100644 --- a/resources/assets/components/AccountImport.vue +++ b/resources/assets/components/AccountImport.vue @@ -381,7 +381,7 @@ let file = this.$refs.zipInput.files[0]; let entries = await this.model(file); if (entries && entries.length) { - let files = await entries.filter(e => e.filename === 'content/posts_1.json'); + let files = await entries.filter(e => e.filename === 'content/posts_1.json' || e.filename === 'your_instagram_activity/content/posts_1.json'); if(!files || !files.length) { this.contactModal( @@ -402,7 +402,7 @@ let entries = await this.model(file); if (entries && entries.length) { this.zipFiles = entries; - let media = await entries.filter(e => e.filename === 'content/posts_1.json')[0].getData(new zip.TextWriter()); + let media = await entries.filter(e => e.filename === 'content/posts_1.json' || e.filename === 'your_instagram_activity/content/posts_1.json')[0].getData(new zip.TextWriter()); this.filterPostMeta(media); let imgs = await Promise.all(entries.filter(entry => { diff --git a/resources/assets/js/components/ComposeModal.vue b/resources/assets/js/components/ComposeModal.vue index 4ffd84666..2c4e3ba42 100644 --- a/resources/assets/js/components/ComposeModal.vue +++ b/resources/assets/js/components/ComposeModal.vue @@ -1204,12 +1204,19 @@ export default { }, 300); }).catch(function(e) { switch(e.response.status) { + case 403: + self.uploading = false; + io.value = null; + swal('Account size limit reached', 'Contact your admin for assistance.', 'error'); + self.page = 2; + break; + case 413: self.uploading = false; io.value = null; swal('File is too large', 'The file you uploaded has the size of ' + self.formatBytes(io.size) + '. Unfortunately, only images up to ' + self.formatBytes(self.config.uploader.max_photo_size * 1024) + ' are supported.\nPlease resize the file and try again.', 'error'); self.page = 2; - break; + break; case 451: self.uploading = false; diff --git a/resources/lang/vendor/backup/ja/notifications.php b/resources/lang/vendor/backup/ja/notifications.php index 911fa4ab2..b8fff4a2b 100644 --- a/resources/lang/vendor/backup/ja/notifications.php +++ b/resources/lang/vendor/backup/ja/notifications.php @@ -31,5 +31,5 @@ return [ 'unhealthy_backup_found_empty' => 'このアプリケーションのバックアップはありません。', 'unhealthy_backup_found_old' => ':date に作成されたバックアップは古すぎます。', 'unhealthy_backup_found_unknown' => '正確な原因が特定できませんでした。', - 'unhealthy_backup_found_full' => 'バックアップが使用できる容量(:disk_limit)を超えています。(現在の使用量 :disk_usage), + 'unhealthy_backup_found_full' => 'バックアップが使用できる容量(:disk_limit)を超えています。(現在の使用量 :disk_usage)', ];