From e5401f8558aefdc80caad75bc604a6b672cd9e83 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 16 Nov 2023 00:29:10 -0700 Subject: [PATCH 1/3] Update StatusHashtagService, remove problemaatic cache layer --- app/Services/StatusHashtagService.php | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/app/Services/StatusHashtagService.php b/app/Services/StatusHashtagService.php index ececca633..6e740db42 100644 --- a/app/Services/StatusHashtagService.php +++ b/app/Services/StatusHashtagService.php @@ -84,18 +84,14 @@ class StatusHashtagService { public static function statusTags($statusId) { - $key = 'pf:services:sh:id:' . $statusId; + $status = Status::with('hashtags')->find($statusId); + if(!$status) { + return []; + } - return Cache::remember($key, 604800, function() use($statusId) { - $status = Status::find($statusId); - if(!$status) { - return []; - } - - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer()); - return $fractal->createData($resource)->toArray(); - }); + $fractal = new Fractal\Manager(); + $fractal->setSerializer(new ArraySerializer()); + $resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer()); + return $fractal->createData($resource)->toArray(); } } From f105f4e8f6335126dbde84836bf8c9ddd9148bb7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 16 Nov 2023 00:47:49 -0700 Subject: [PATCH 2/3] Update HomeFeedPipeline, fix tag filtering --- .editorconfig | 2 +- .../HashtagInsertFanoutPipeline.php | 13 +- .../HashtagRemoveFanoutPipeline.php | 10 + .../HashtagUnfollowPipeline.php | 10 +- app/Jobs/StatusPipeline/StatusEntityLexer.php | 290 +++++++++--------- .../StatusPipeline/StatusTagsPipeline.php | 184 +++++------ 6 files changed, 267 insertions(+), 242 deletions(-) diff --git a/.editorconfig b/.editorconfig index 1cd7d1077..3c44241cc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,8 +1,8 @@ root = true [*] +indent_style = space indent_size = 4 -indent_style = tab end_of_line = lf charset = utf-8 trim_trailing_whitespace = true diff --git a/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php b/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php index 64adebadc..581b8784f 100644 --- a/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php +++ b/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php @@ -12,6 +12,7 @@ use App\Hashtag; use App\StatusHashtag; use App\Services\HashtagFollowService; use App\Services\HomeTimelineService; +use App\Services\StatusService; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; @@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro public function handle(): void { $hashtag = $this->hashtag; + $sid = $hashtag->status_id; + $status = StatusService::get($sid, false); + + if(!$status) { + return; + } + + if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) { + return; + } $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id); if(!$ids || !count($ids)) { - return; + return; } foreach($ids as $id) { diff --git a/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php b/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php index 92e3b8e42..ea9d87fbb 100644 --- a/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php +++ b/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php @@ -12,6 +12,7 @@ use App\Hashtag; use App\StatusHashtag; use App\Services\HashtagFollowService; use App\Services\HomeTimelineService; +use App\Services\StatusService; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; @@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro { $sid = $this->sid; $hid = $this->hid; + $status = StatusService::get($sid, false); + + if(!$status) { + return; + } + + if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) { + return; + } $ids = HashtagFollowService::getPidByHid($hid); diff --git a/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php b/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php index de2af9132..232179ec3 100644 --- a/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php +++ b/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php @@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue { $hid = $this->hid; $pid = $this->pid; - $slug = $this->slug; + $slug = strtolower($this->slug); $statusIds = HomeTimelineService::get($pid, 0, -1); @@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue foreach($statusIds as $id) { $status = StatusService::get($id, false); - if(!$status) { + if(!$status || empty($status['tags'])) { HomeTimelineService::rem($pid, $id); continue; } - $following = in_array($status['account']['id'], $followingIds); - if($following || !isset($status['tags'])) { + $following = in_array((int) $status['account']['id'], $followingIds); + if($following === true) { continue; } $tags = collect($status['tags'])->map(function($tag) { - return $tag['name']; + return strtolower($tag['name']); })->filter()->values()->toArray(); if(in_array($slug, $tags)) { diff --git a/app/Jobs/StatusPipeline/StatusEntityLexer.php b/app/Jobs/StatusPipeline/StatusEntityLexer.php index a6266044c..872594a96 100644 --- a/app/Jobs/StatusPipeline/StatusEntityLexer.php +++ b/app/Jobs/StatusPipeline/StatusEntityLexer.php @@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use App\Services\StatusService; use App\Services\UserFilterService; use App\Services\AdminShadowFilterService; use App\Jobs\HomeFeedPipeline\FeedInsertPipeline; @@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline; class StatusEntityLexer implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $status; - protected $entities; - protected $autolink; + protected $status; + protected $entities; + protected $autolink; - /** - * Delete the job if its models no longer exist. - * - * @var bool - */ - public $deleteWhenMissingModels = true; + /** + * Delete the job if its models no longer exist. + * + * @var bool + */ + public $deleteWhenMissingModels = true; - /** - * Create a new job instance. - * - * @return void - */ - public function __construct(Status $status) - { - $this->status = $status; - } + /** + * Create a new job instance. + * + * @return void + */ + public function __construct(Status $status) + { + $this->status = $status; + } - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - $profile = $this->status->profile; - $status = $this->status; + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $profile = $this->status->profile; + $status = $this->status; - if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) { - $profile->status_count = $profile->status_count + 1; - $profile->save(); - } + if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) { + $profile->status_count = $profile->status_count + 1; + $profile->save(); + } - if($profile->no_autolink == false) { - $this->parseEntities(); - } - } + if($profile->no_autolink == false) { + $this->parseEntities(); + } + } - public function parseEntities() - { - $this->extractEntities(); - } + public function parseEntities() + { + $this->extractEntities(); + } - public function extractEntities() - { - $this->entities = Extractor::create()->extract($this->status->caption); - $this->autolinkStatus(); - } + public function extractEntities() + { + $this->entities = Extractor::create()->extract($this->status->caption); + $this->autolinkStatus(); + } - public function autolinkStatus() - { - $this->autolink = Autolink::create()->autolink($this->status->caption); - $this->storeEntities(); - } + public function autolinkStatus() + { + $this->autolink = Autolink::create()->autolink($this->status->caption); + $this->storeEntities(); + } - public function storeEntities() - { - $this->storeHashtags(); - DB::transaction(function () { - $status = $this->status; - $status->rendered = nl2br($this->autolink); - $status->save(); - }); - } + public function storeEntities() + { + $this->storeHashtags(); + DB::transaction(function () { + $status = $this->status; + $status->rendered = nl2br($this->autolink); + $status->save(); + }); + } - public function storeHashtags() - { - $tags = array_unique($this->entities['hashtags']); - $status = $this->status; + public function storeHashtags() + { + $tags = array_unique($this->entities['hashtags']); + $status = $this->status; - foreach ($tags as $tag) { - if(mb_strlen($tag) > 124) { - continue; - } - DB::transaction(function () use ($status, $tag) { - $slug = str_slug($tag, '-', false); + foreach ($tags as $tag) { + if(mb_strlen($tag) > 124) { + continue; + } + DB::transaction(function () use ($status, $tag) { + $slug = str_slug($tag, '-', false); $hashtag = Hashtag::firstOrCreate([ 'slug' => $slug @@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue 'name' => $tag ]); - StatusHashtag::firstOrCreate( - [ - 'status_id' => $status->id, - 'hashtag_id' => $hashtag->id, - 'profile_id' => $status->profile_id, - 'status_visibility' => $status->visibility, - ] - ); - }); - } - $this->storeMentions(); - } + StatusHashtag::firstOrCreate( + [ + 'status_id' => $status->id, + 'hashtag_id' => $hashtag->id, + 'profile_id' => $status->profile_id, + 'status_visibility' => $status->visibility, + ] + ); + }); + } + $this->storeMentions(); + } - public function storeMentions() - { - $mentions = array_unique($this->entities['mentions']); - $status = $this->status; + public function storeMentions() + { + $mentions = array_unique($this->entities['mentions']); + $status = $this->status; - foreach ($mentions as $mention) { - $mentioned = Profile::whereUsername($mention)->first(); + foreach ($mentions as $mention) { + $mentioned = Profile::whereUsername($mention)->first(); - if (empty($mentioned) || !isset($mentioned->id)) { - continue; - } + if (empty($mentioned) || !isset($mentioned->id)) { + continue; + } $blocks = UserFilterService::blocks($mentioned->id); if($blocks && in_array($status->profile_id, $blocks)) { continue; } - DB::transaction(function () use ($status, $mentioned) { - $m = new Mention(); - $m->status_id = $status->id; - $m->profile_id = $mentioned->id; - $m->save(); + DB::transaction(function () use ($status, $mentioned) { + $m = new Mention(); + $m->status_id = $status->id; + $m->profile_id = $mentioned->id; + $m->save(); - MentionPipeline::dispatch($status, $m); - }); - } - $this->fanout(); - } + MentionPipeline::dispatch($status, $m); + }); + } + $this->fanout(); + } - public function fanout() - { - $status = $this->status; + public function fanout() + { + $status = $this->status; + StatusService::refresh($status->id); - if(config('exp.cached_home_timeline')) { - if( $status->in_reply_to_id === null && - in_array($status->scope, ['public', 'unlisted', 'private']) - ) { - FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed'); - } - } - $this->deliver(); - } - - public function deliver() - { - $status = $this->status; - $types = [ - 'photo', - 'photo:album', - 'video', - 'video:album', - 'photo:video:album' - ]; - - if(config_cache('pixelfed.bouncer.enabled')) { - Bouncer::get($status); - } - - Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id); - $hideNsfw = config('instance.hide_nsfw_on_public_feeds'); - if( $status->uri == null && - $status->scope == 'public' && - in_array($status->type, $types) && - $status->in_reply_to_id === null && - $status->reblog_of_id === null && - ($hideNsfw ? $status->is_nsfw == false : true) - ) { - if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) { - PublicTimelineService::add($status->id); + if(config('exp.cached_home_timeline')) { + if( $status->in_reply_to_id === null && + in_array($status->scope, ['public', 'unlisted', 'private']) + ) { + FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed'); } - } + } + $this->deliver(); + } - if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') { - StatusActivityPubDeliver::dispatch($status); - } - } + public function deliver() + { + $status = $this->status; + $types = [ + 'photo', + 'photo:album', + 'video', + 'video:album', + 'photo:video:album' + ]; + + if(config_cache('pixelfed.bouncer.enabled')) { + Bouncer::get($status); + } + + Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id); + $hideNsfw = config('instance.hide_nsfw_on_public_feeds'); + if( $status->uri == null && + $status->scope == 'public' && + in_array($status->type, $types) && + $status->in_reply_to_id === null && + $status->reblog_of_id === null && + ($hideNsfw ? $status->is_nsfw == false : true) + ) { + if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) { + PublicTimelineService::add($status->id); + } + } + + if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') { + StatusActivityPubDeliver::dispatch($status); + } + } } diff --git a/app/Jobs/StatusPipeline/StatusTagsPipeline.php b/app/Jobs/StatusPipeline/StatusTagsPipeline.php index 893fa6a83..003196e0d 100644 --- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php +++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php @@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers; class StatusTagsPipeline implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $activity; - protected $status; + protected $activity; + protected $status; - /** - * Create a new job instance. - * - * @return void - */ - public function __construct($activity, $status) - { - $this->activity = $activity; - $this->status = $status; - } + /** + * Create a new job instance. + * + * @return void + */ + public function __construct($activity, $status) + { + $this->activity = $activity; + $this->status = $status; + } - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - $res = $this->activity; - $status = $this->status; + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $res = $this->activity; + $status = $this->status; if(isset($res['tag']['type'], $res['tag']['name'])) { $res['tag'] = [$res['tag']]; } - $tags = collect($res['tag']); + $tags = collect($res['tag']); - // Emoji - $tags->filter(function($tag) { - return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji'; - }) - ->map(function($tag) { - CustomEmojiService::import($tag['id'], $this->status->id); - }); + // Emoji + $tags->filter(function($tag) { + return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji'; + }) + ->map(function($tag) { + CustomEmojiService::import($tag['id'], $this->status->id); + }); - // Hashtags - $tags->filter(function($tag) { - return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']); - }) - ->map(function($tag) use($status) { - $name = substr($tag['name'], 0, 1) == '#' ? - substr($tag['name'], 1) : $tag['name']; + // Hashtags + $tags->filter(function($tag) { + return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']); + }) + ->map(function($tag) use($status) { + $name = substr($tag['name'], 0, 1) == '#' ? + substr($tag['name'], 1) : $tag['name']; - $banned = TrendingHashtagService::getBannedHashtagNames(); + $banned = TrendingHashtagService::getBannedHashtagNames(); - if(count($banned)) { + if(count($banned)) { if(in_array(strtolower($name), array_map('strtolower', $banned))) { - return; + return; } } if(config('database.default') === 'pgsql') { - $hashtag = Hashtag::where('name', 'ilike', $name) - ->orWhere('slug', 'ilike', str_slug($name, '-', false)) - ->first(); + $hashtag = Hashtag::where('name', 'ilike', $name) + ->orWhere('slug', 'ilike', str_slug($name, '-', false)) + ->first(); - if(!$hashtag) { - $hashtag = Hashtag::updateOrCreate([ - 'slug' => str_slug($name, '-', false), - 'name' => $name - ]); - } + if(!$hashtag) { + $hashtag = Hashtag::updateOrCreate([ + 'slug' => str_slug($name, '-', false), + 'name' => $name + ]); + } } else { - $hashtag = Hashtag::updateOrCreate([ - 'slug' => str_slug($name, '-', false), - 'name' => $name - ]); + $hashtag = Hashtag::updateOrCreate([ + 'slug' => str_slug($name, '-', false), + 'name' => $name + ]); } - StatusHashtag::firstOrCreate([ - 'status_id' => $status->id, - 'hashtag_id' => $hashtag->id, - 'profile_id' => $status->profile_id, - 'status_visibility' => $status->scope - ]); - }); + StatusHashtag::firstOrCreate([ + 'status_id' => $status->id, + 'hashtag_id' => $hashtag->id, + 'profile_id' => $status->profile_id, + 'status_visibility' => $status->scope + ]); + }); - // Mentions - $tags->filter(function($tag) { - return $tag && - $tag['type'] == 'Mention' && - isset($tag['href']) && - substr($tag['href'], 0, 8) === 'https://'; - }) - ->map(function($tag) use($status) { - if(Helpers::validateLocalUrl($tag['href'])) { - $parts = explode('/', $tag['href']); - if(!$parts) { - return; - } - $pid = AccountService::usernameToId(end($parts)); - if(!$pid) { - return; - } - } else { - $acct = Helpers::profileFetch($tag['href']); - if(!$acct) { - return; - } - $pid = $acct->id; - } - $mention = new Mention; - $mention->status_id = $status->id; - $mention->profile_id = $pid; - $mention->save(); - MentionPipeline::dispatch($status, $mention); - }); - } + // Mentions + $tags->filter(function($tag) { + return $tag && + $tag['type'] == 'Mention' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }) + ->map(function($tag) use($status) { + if(Helpers::validateLocalUrl($tag['href'])) { + $parts = explode('/', $tag['href']); + if(!$parts) { + return; + } + $pid = AccountService::usernameToId(end($parts)); + if(!$pid) { + return; + } + } else { + $acct = Helpers::profileFetch($tag['href']); + if(!$acct) { + return; + } + $pid = $acct->id; + } + $mention = new Mention; + $mention->status_id = $status->id; + $mention->profile_id = $pid; + $mention->save(); + MentionPipeline::dispatch($status, $mention); + }); + + StatusService::refresh($status->id); + } } From e6d3c7f4d7838cc7afb26ec82210497eb76b9f4c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 16 Nov 2023 00:49:51 -0700 Subject: [PATCH 3/3] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f130ce819..505df79bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,8 @@ - Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885)) - Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae)) - Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008)) +- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85)) +- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)