From 6d6a033a54e4a058abff3ebf09065d5fc48e8820 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Wed, 31 Aug 2022 20:57:20 -0600 Subject: [PATCH 1/8] Update DeleteWorker, remove cache lock --- app/Jobs/InboxPipeline/DeleteWorker.php | 50 +++++++++++-------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/app/Jobs/InboxPipeline/DeleteWorker.php b/app/Jobs/InboxPipeline/DeleteWorker.php index a55623e63..5639f9752 100644 --- a/app/Jobs/InboxPipeline/DeleteWorker.php +++ b/app/Jobs/InboxPipeline/DeleteWorker.php @@ -72,39 +72,31 @@ class DeleteWorker implements ShouldQueue 'b:' . base64_encode($actor) : 'h:' . hash('sha256', $actor); - $lockKey = 'ap:inbox:actor-delete-exists:lock:' . $hash; - Cache::lock($lockKey, 30)->block(15, function () use( - $headers, - $payload, - $actor, - $hash - ) { - $key = 'ap:inbox:actor-delete-exists:' . $hash; - $actorDelete = Cache::remember($key, now()->addMinutes(15), function() use($actor) { - return Profile::whereRemoteUrl($actor) - ->whereNotNull('domain') - ->exists(); - }); - if($actorDelete) { - if($this->verifySignature($headers, $payload) == true) { - Cache::set($key, false); - $profile = Profile::whereNotNull('domain') - ->whereNull('status') - ->whereRemoteUrl($actor) - ->first(); - if($profile) { - DeleteRemoteProfilePipeline::dispatch($profile)->onQueue('delete'); - } - return 1; - } else { - // Signature verification failed, exit. - return 1; + $key = 'ap:inbox:actor-delete-exists:' . $hash; + $actorDelete = Cache::remember($key, now()->addMinutes(15), function() use($actor) { + return Profile::whereRemoteUrl($actor) + ->whereNotNull('domain') + ->exists(); + }); + if($actorDelete) { + if($this->verifySignature($headers, $payload) == true) { + Cache::set($key, false); + $profile = Profile::whereNotNull('domain') + ->whereNull('status') + ->whereRemoteUrl($actor) + ->first(); + if($profile) { + DeleteRemoteProfilePipeline::dispatch($profile)->onQueue('delete'); } + return 1; } else { - // Remote user doesn't exist, exit early. + // Signature verification failed, exit. return 1; } - }); + } else { + // Remote user doesn't exist, exit early. + return 1; + } return 1; } From 9d31f73bfaa664faa05d7ab9846a4f6bb8a871b2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Wed, 31 Aug 2022 21:37:43 -0600 Subject: [PATCH 2/8] Fix SearchApiV2Service, improve webfinger condition --- app/Services/SearchApiV2Service.php | 3 +-- app/Services/WebfingerService.php | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Services/SearchApiV2Service.php b/app/Services/SearchApiV2Service.php index 575cfe0da..70877f0c6 100644 --- a/app/Services/SearchApiV2Service.php +++ b/app/Services/SearchApiV2Service.php @@ -33,7 +33,6 @@ class SearchApiV2Service $q = urldecode($query->input('q')); if($query->has('resolve') && - $query->resolve == true && ( Str::startsWith($q, 'https://') || Str::substrCount($q, '@') >= 1) ) { @@ -203,7 +202,7 @@ class SearchApiV2Service return $default; } - if(Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) { + if(!Str::startsWith($query, 'http') && Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) { try { $res = WebfingerService::lookup('@' . $query, $mastodonMode); } catch (\Exception $e) { diff --git a/app/Services/WebfingerService.php b/app/Services/WebfingerService.php index fd5f81d7f..807914664 100644 --- a/app/Services/WebfingerService.php +++ b/app/Services/WebfingerService.php @@ -56,6 +56,9 @@ class WebfingerService ->first(); $profile = Helpers::profileFetch($link); + if(!$profile) { + return; + } return $mastodonMode ? AccountService::getMastodon($profile->id, true) : AccountService::get($profile->id); From 2c20d9e3984abee6ed0cb042de12e4d37b3bf248 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Wed, 31 Aug 2022 21:41:17 -0600 Subject: [PATCH 3/8] Update inbox handler, upsert statuses to fix duplicate bug. Fixes #2670, #2961, #3556 --- app/Status.php | 19 ++++++++++++++++- app/Util/ActivityPub/Helpers.php | 35 +++++++++++++++++--------------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/app/Status.php b/app/Status.php index ba1614a60..e944ac242 100644 --- a/app/Status.php +++ b/app/Status.php @@ -28,7 +28,24 @@ class Status extends Model */ protected $dates = ['deleted_at']; - protected $fillable = ['profile_id', 'visibility', 'in_reply_to_id', 'reblog_of_id', 'type']; + protected $fillable = [ + 'scope', + 'caption', + 'rendered', + 'url', + 'uri', + 'object_url', + 'created_at', + 'local', + 'is_nsfw', + 'scope', + 'cw_summary', + 'profile_id', + 'visibility', + 'in_reply_to_id', + 'reblog_of_id', + 'type' + ]; const STATUS_TYPES = [ 'text', diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 1a210b056..3f41b7e22 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -474,22 +474,25 @@ class Helpers { return; } - $status = new Status; - $status->profile_id = $pid; - $status->url = $url; - $status->uri = $url; - $status->object_url = $id; - $status->caption = strip_tags($activity['content']); - $status->rendered = Purify::clean($activity['content']); - $status->created_at = Carbon::parse($ts)->tz('UTC'); - $status->in_reply_to_id = $reply_to; - $status->local = false; - $status->is_nsfw = $cw; - $status->scope = $scope; - $status->visibility = $scope; - $status->cw_summary = $cw == true && isset($activity['summary']) ? - Purify::clean(strip_tags($activity['summary'])) : null; - $status->save(); + $status = Status::updateOrCreate( + [ + 'uri' => $url + ], [ + 'profile_id' => $pid, + 'url' => $url, + 'object_url' => $id, + 'caption' => strip_tags($activity['content']), + 'rendered' => Purify::clean($activity['content']), + 'created_at' => Carbon::parse($ts)->tz('UTC'), + 'in_reply_to_id' => $reply_to, + 'local' => false, + 'is_nsfw' => $cw, + 'scope' => $scope, + 'visibility' => $scope, + 'cw_summary' => ($cw == true && isset($activity['summary']) ? + Purify::clean(strip_tags($activity['summary'])) : null) + ] + ); if($reply_to == null) { self::importNoteAttachment($activity, $status); From bc2bbc14accd8b4ccd570623d38a33b5fb926ca3 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Wed, 31 Aug 2022 22:02:41 -0600 Subject: [PATCH 4/8] Update AP helpers, remove cache lock from profileUpdateOrCreate method --- app/Util/ActivityPub/Helpers.php | 123 ++++++++++++++----------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 3f41b7e22..cdb3b225a 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -720,75 +720,66 @@ class Helpers { public static function profileUpdateOrCreate($url) { - $hash = base64_encode($url); - $key = 'ap:profile:by_url:' . $hash; - $lock = Cache::lock($key, 30); - $profile = null; - - try { - $lock->block(5); - - $res = self::fetchProfileFromUrl($url); - if(isset($res['id']) == false) { - return; - } - $domain = parse_url($res['id'], PHP_URL_HOST); - if(!isset($res['preferredUsername']) && !isset($res['nickname'])) { - return; - } - $username = (string) Purify::clean($res['preferredUsername'] ?? $res['nickname']); - if(empty($username)) { - return; - } - $remoteUsername = $username; - $webfinger = "@{$username}@{$domain}"; - - abort_if(!self::validateUrl($res['inbox']), 400); - abort_if(!self::validateUrl($res['id']), 400); - - $profile = DB::transaction(function() use($domain, $webfinger, $res) { - $instance = Instance::updateOrCreate([ - 'domain' => $domain - ]); - if($instance->wasRecentlyCreated == true) { - \App\Jobs\InstancePipeline\FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); - } - - $profile = Profile::updateOrCreate( - [ - 'domain' => strtolower($domain), - 'username' => Purify::clean($webfinger), - 'remote_url' => $res['id'], - ], - [ - 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', - 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, - 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, - 'inbox_url' => $res['inbox'], - 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, - 'public_key' => $res['publicKey']['publicKeyPem'], - 'key_id' => $res['publicKey']['id'], - 'webfinger' => Purify::clean($webfinger), - ] - ); - - if( $profile->last_fetched_at == null || - $profile->last_fetched_at->lt(now()->subHours(24)) - ) { - RemoteAvatarFetch::dispatch($profile); - } - $profile->last_fetched_at = now(); - $profile->save(); - return $profile; - }); - - return $profile; - } catch (LockTimeoutException $e) { - } finally { - optional($lock)->release(); + $res = self::fetchProfileFromUrl($url); + if(!$res || isset($res['id']) == false) { + return; } + $domain = parse_url($res['id'], PHP_URL_HOST); + if(!isset($res['preferredUsername']) && !isset($res['nickname'])) { + return; + } + $username = (string) Purify::clean($res['preferredUsername'] ?? $res['nickname']); + if(empty($username)) { + return; + } + $remoteUsername = $username; + $webfinger = "@{$username}@{$domain}"; + + if(!self::validateUrl($res['inbox'])) { + return; + } + if(!self::validateUrl($res['id'])) { + return; + } + + $profile = DB::transaction(function() use($domain, $webfinger, $res) { + $instance = Instance::updateOrCreate([ + 'domain' => $domain + ]); + if($instance->wasRecentlyCreated == true) { + \App\Jobs\InstancePipeline\FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); + } + + $profile = Profile::updateOrCreate( + [ + 'domain' => strtolower($domain), + 'username' => Purify::clean($webfinger), + 'remote_url' => $res['id'], + 'webfinger' => Purify::clean($webfinger), + 'key_id' => $res['publicKey']['id'], + ], + [ + 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', + 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, + 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, + 'inbox_url' => $res['inbox'], + 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, + 'public_key' => $res['publicKey']['publicKeyPem'], + ] + ); + + if( $profile->last_fetched_at == null || + $profile->last_fetched_at->lt(now()->subHours(24)) + ) { + RemoteAvatarFetch::dispatch($profile); + } + $profile->last_fetched_at = now(); + $profile->save(); + return $profile; + }); return $profile; + } public static function profileFetch($url) From 66aa8bf9dec7c709d9a6e961d851f481faaac35e Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:13:01 -0600 Subject: [PATCH 5/8] Add migrations to fix webfinger profiles --- ...x_webfinger_profile_duplicate_accounts.php | 59 +++++++++++++++++++ ...002_generate_missing_profile_webfinger.php | 38 ++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 database/migrations/2022_09_01_000000_fix_webfinger_profile_duplicate_accounts.php create mode 100644 database/migrations/2022_09_01_043002_generate_missing_profile_webfinger.php diff --git a/database/migrations/2022_09_01_000000_fix_webfinger_profile_duplicate_accounts.php b/database/migrations/2022_09_01_000000_fix_webfinger_profile_duplicate_accounts.php new file mode 100644 index 000000000..3a10c9d5f --- /dev/null +++ b/database/migrations/2022_09_01_000000_fix_webfinger_profile_duplicate_accounts.php @@ -0,0 +1,59 @@ +where('username', 'not like', '@%') + ->chunk(200, function($profiles) { + foreach($profiles as $profile) { + $exists = Profile::whereUsername("@{$profile->username}@{$profile->domain}")->first(); + if($exists) { + $exists->username = null; + $exists->domain = null; + $exists->webfinger = null; + $exists->save(); + DeleteRemoteProfilePipeline::dispatch($exists); + + $profile->username = "@{$profile->username}@{$profile->domain}"; + if(!$profile->webfinger) { + $profile->webfinger = "@{$profile->username}@{$profile->domain}"; + } + $profile->save(); + } else { + $profile->username = "@{$profile->username}@{$profile->domain}"; + if(!$profile->webfinger) { + $profile->webfinger = "@{$profile->username}@{$profile->domain}"; + } + $profile->save(); + } + } + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + // + } +} diff --git a/database/migrations/2022_09_01_043002_generate_missing_profile_webfinger.php b/database/migrations/2022_09_01_043002_generate_missing_profile_webfinger.php new file mode 100644 index 000000000..55a0f9002 --- /dev/null +++ b/database/migrations/2022_09_01_043002_generate_missing_profile_webfinger.php @@ -0,0 +1,38 @@ +whereNull('webfinger') + ->chunk(200, function($profiles) { + foreach($profiles as $profile) { + if(substr($profile->username, 0, 1) === "@") { + $profile->webfinger = $profile->username; + $profile->save(); + } + } + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + // + } +} From acd8f5bb49fa9b4199e17e1e1b356fcc4e000935 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:13:57 -0600 Subject: [PATCH 6/8] Update ap helpers, move remote_url constraint --- app/Util/ActivityPub/Helpers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index cdb3b225a..dcfa02e98 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -754,11 +754,11 @@ class Helpers { [ 'domain' => strtolower($domain), 'username' => Purify::clean($webfinger), - 'remote_url' => $res['id'], 'webfinger' => Purify::clean($webfinger), 'key_id' => $res['publicKey']['id'], ], [ + 'remote_url' => $res['id'], 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, From 8bcf3700baa05cb3b0239b51e301c140fd0ae9f4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:21:06 -0600 Subject: [PATCH 7/8] Update status model --- app/Status.php | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/app/Status.php b/app/Status.php index e944ac242..6150a9651 100644 --- a/app/Status.php +++ b/app/Status.php @@ -28,24 +28,7 @@ class Status extends Model */ protected $dates = ['deleted_at']; - protected $fillable = [ - 'scope', - 'caption', - 'rendered', - 'url', - 'uri', - 'object_url', - 'created_at', - 'local', - 'is_nsfw', - 'scope', - 'cw_summary', - 'profile_id', - 'visibility', - 'in_reply_to_id', - 'reblog_of_id', - 'type' - ]; + protected $guarded = []; const STATUS_TYPES = [ 'text', From e11d1336c66e3b69f2f86c102491a04e14e0cafe Mon Sep 17 00:00:00 2001 From: Daniel Supernault <877217+dansup@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:21:48 -0600 Subject: [PATCH 8/8] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8638c4174..7533bf1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,12 @@ - Fix cache invalidation in AdminSettingsController when updating rules ([fe6787f7](https://github.com/pixelfed/pixelfed/commit/fe6787f7)) - Update SearchApiService, improve account/webfinger results ([533f7165](https://github.com/pixelfed/pixelfed/commit/533f7165)) - Update NotificationService, fix account attribute ([949b7bb6](https://github.com/pixelfed/pixelfed/commit/949b7bb6)) +- Update DeleteWorker, remove cache lock ([6d6a033a](https://github.com/pixelfed/pixelfed/commit/6d6a033a)) +- Fix SearchApiV2Service, improve webfinger condition ([9d31f73b](https://github.com/pixelfed/pixelfed/commit/9d31f73b)) +- Update inbox handler, upsert statuses to fix duplicate bug. Fixes #2670, #2961, #3556 ([2c20d9e3](https://github.com/pixelfed/pixelfed/commit/2c20d9e3)) +- Update AP helpers, remove cache lock from profileUpdateOrCreate method and move webfinger + key_id to unique constraints to fix sql duplicate errors ([bc2bbc14](https://github.com/pixelfed/pixelfed/commit/bc2bbc14)) +- Add migrations to fix webfinger profiles ([66aa8bf9](https://github.com/pixelfed/pixelfed/commit/66aa8bf9)) +- Update ap helpers, move remote_url constraint ([acd8f5bb](https://github.com/pixelfed/pixelfed/commit/acd8f5bb)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3)