Merge pull request #4774 from pixelfed/staging

Staging
2024-11-22 06:21:27 +00:00 · 2023-11-16 00:50:09 -07:00 · 2023-11-16 00:50:09 -07:00 · 7f6d64b517
commit 7f6d64b517
parent dfbc453b03 e6d3c7f4d7
8 changed files with 277 additions and 254 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -1,8 +1,8 @@
 root = true

 [*]
+indent_style = space
 indent_size = 4
-indent_style = tab
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -54,6 +54,8 @@
 - Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885))
 - Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae))
 - Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008))
+- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
+- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))

 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)
--- a/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php
+++ b/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php
@ -12,6 +12,7 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\Services\HashtagFollowService;
 use App\Services\HomeTimelineService;
+use App\Services\StatusService;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;

@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
    public function handle(): void
    {
        $hashtag = $this->hashtag;
+        $sid = $hashtag->status_id;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }

        $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);

        if(!$ids || !count($ids)) {
-        	return;
+            return;
        }

        foreach($ids as $id) {
--- a/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php
+++ b/app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php
@ -12,6 +12,7 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\Services\HashtagFollowService;
 use App\Services\HomeTimelineService;
+use App\Services\StatusService;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;

@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
    {
        $sid = $this->sid;
        $hid = $this->hid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }

        $ids = HashtagFollowService::getPidByHid($hid);

--- a/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php
+++ b/app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php
@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
    {
        $hid = $this->hid;
        $pid = $this->pid;
-        $slug = $this->slug;
+        $slug = strtolower($this->slug);

        $statusIds = HomeTimelineService::get($pid, 0, -1);

@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue

        foreach($statusIds as $id) {
            $status = StatusService::get($id, false);
-            if(!$status) {
+            if(!$status || empty($status['tags'])) {
                HomeTimelineService::rem($pid, $id);
                continue;
            }
-            $following = in_array($status['account']['id'], $followingIds);
-            if($following || !isset($status['tags'])) {
+            $following = in_array((int) $status['account']['id'], $followingIds);
+            if($following === true) {
                continue;
            }

            $tags = collect($status['tags'])->map(function($tag) {
-                return $tag['name'];
+                return strtolower($tag['name']);
            })->filter()->values()->toArray();

            if(in_array($slug, $tags)) {
--- a/app/Jobs/StatusPipeline/StatusEntityLexer.php
+++ b/app/Jobs/StatusPipeline/StatusEntityLexer.php
@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use App\Services\StatusService;
 use App\Services\UserFilterService;
 use App\Services\AdminShadowFilterService;
 use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;

 class StatusEntityLexer implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

-	protected $status;
-	protected $entities;
-	protected $autolink;
+    protected $status;
+    protected $entities;
+    protected $autolink;

-	/**
-	 * Delete the job if its models no longer exist.
-	 *
-	 * @var bool
-	 */
-	public $deleteWhenMissingModels = true;
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;

-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct(Status $status)
-	{
-		$this->status = $status;
-	}
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Status $status)
+    {
+        $this->status = $status;
+    }

-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$profile = $this->status->profile;
-		$status = $this->status;
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $profile = $this->status->profile;
+        $status = $this->status;

-		if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
-			$profile->status_count = $profile->status_count + 1;
-			$profile->save();
-		}
+        if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            $profile->status_count = $profile->status_count + 1;
+            $profile->save();
+        }

-		if($profile->no_autolink == false) {
-			$this->parseEntities();
-		}
-	}
+        if($profile->no_autolink == false) {
+            $this->parseEntities();
+        }
+    }

-	public function parseEntities()
-	{
-		$this->extractEntities();
-	}
+    public function parseEntities()
+    {
+        $this->extractEntities();
+    }

-	public function extractEntities()
-	{
-		$this->entities = Extractor::create()->extract($this->status->caption);
-		$this->autolinkStatus();
-	}
+    public function extractEntities()
+    {
+        $this->entities = Extractor::create()->extract($this->status->caption);
+        $this->autolinkStatus();
+    }

-	public function autolinkStatus()
-	{
-		$this->autolink = Autolink::create()->autolink($this->status->caption);
-		$this->storeEntities();
-	}
+    public function autolinkStatus()
+    {
+        $this->autolink = Autolink::create()->autolink($this->status->caption);
+        $this->storeEntities();
+    }

-	public function storeEntities()
-	{
-		$this->storeHashtags();
-		DB::transaction(function () {
-			$status = $this->status;
-			$status->rendered = nl2br($this->autolink);
-			$status->save();
-		});
-	}
+    public function storeEntities()
+    {
+        $this->storeHashtags();
+        DB::transaction(function () {
+            $status = $this->status;
+            $status->rendered = nl2br($this->autolink);
+            $status->save();
+        });
+    }

-	public function storeHashtags()
-	{
-		$tags = array_unique($this->entities['hashtags']);
-		$status = $this->status;
+    public function storeHashtags()
+    {
+        $tags = array_unique($this->entities['hashtags']);
+        $status = $this->status;

-		foreach ($tags as $tag) {
-			if(mb_strlen($tag) > 124) {
-				continue;
-			}
-			DB::transaction(function () use ($status, $tag) {
-				$slug = str_slug($tag, '-', false);
+        foreach ($tags as $tag) {
+            if(mb_strlen($tag) > 124) {
+                continue;
+            }
+            DB::transaction(function () use ($status, $tag) {
+                $slug = str_slug($tag, '-', false);

                $hashtag = Hashtag::firstOrCreate([
                    'slug' => $slug
@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue
                    'name' => $tag
                ]);

-				StatusHashtag::firstOrCreate(
-					[
-						'status_id' => $status->id,
-						'hashtag_id' => $hashtag->id,
-						'profile_id' => $status->profile_id,
-						'status_visibility' => $status->visibility,
-					]
-				);
-			});
-		}
-		$this->storeMentions();
-	}
+                StatusHashtag::firstOrCreate(
+                    [
+                        'status_id' => $status->id,
+                        'hashtag_id' => $hashtag->id,
+                        'profile_id' => $status->profile_id,
+                        'status_visibility' => $status->visibility,
+                    ]
+                );
+            });
+        }
+        $this->storeMentions();
+    }

-	public function storeMentions()
-	{
-		$mentions = array_unique($this->entities['mentions']);
-		$status = $this->status;
+    public function storeMentions()
+    {
+        $mentions = array_unique($this->entities['mentions']);
+        $status = $this->status;

-		foreach ($mentions as $mention) {
-			$mentioned = Profile::whereUsername($mention)->first();
+        foreach ($mentions as $mention) {
+            $mentioned = Profile::whereUsername($mention)->first();

-			if (empty($mentioned) || !isset($mentioned->id)) {
-				continue;
-			}
+            if (empty($mentioned) || !isset($mentioned->id)) {
+                continue;
+            }
            $blocks = UserFilterService::blocks($mentioned->id);
            if($blocks && in_array($status->profile_id, $blocks)) {
                continue;
            }

-			DB::transaction(function () use ($status, $mentioned) {
-				$m = new Mention();
-				$m->status_id = $status->id;
-				$m->profile_id = $mentioned->id;
-				$m->save();
+            DB::transaction(function () use ($status, $mentioned) {
+                $m = new Mention();
+                $m->status_id = $status->id;
+                $m->profile_id = $mentioned->id;
+                $m->save();

-				MentionPipeline::dispatch($status, $m);
-			});
-		}
-		$this->fanout();
-	}
+                MentionPipeline::dispatch($status, $m);
+            });
+        }
+        $this->fanout();
+    }

-	public function fanout()
-	{
-		$status = $this->status;
+    public function fanout()
+    {
+        $status = $this->status;
+        StatusService::refresh($status->id);

-		if(config('exp.cached_home_timeline')) {
-			if( $status->in_reply_to_id === null &&
-				in_array($status->scope, ['public', 'unlisted', 'private'])
-			) {
-				FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
-			}
-		}
-		$this->deliver();
-	}
-
-	public function deliver()
-	{
-		$status = $this->status;
-		$types = [
-			'photo',
-			'photo:album',
-			'video',
-			'video:album',
-			'photo:video:album'
-		];
-
-		if(config_cache('pixelfed.bouncer.enabled')) {
-			Bouncer::get($status);
-		}
-
-		Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
-		$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
-		if( $status->uri == null &&
-			$status->scope == 'public' &&
-			in_array($status->type, $types) &&
-			$status->in_reply_to_id === null &&
-			$status->reblog_of_id === null &&
-			($hideNsfw ? $status->is_nsfw == false : true)
-		) {
-            if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
-    			PublicTimelineService::add($status->id);
+        if(config('exp.cached_home_timeline')) {
+            if( $status->in_reply_to_id === null &&
+                in_array($status->scope, ['public', 'unlisted', 'private'])
+            ) {
+                FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
            }
-		}
+        }
+        $this->deliver();
+    }

-		if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
-			StatusActivityPubDeliver::dispatch($status);
-		}
-	}
+    public function deliver()
+    {
+        $status = $this->status;
+        $types = [
+            'photo',
+            'photo:album',
+            'video',
+            'video:album',
+            'photo:video:album'
+        ];
+
+        if(config_cache('pixelfed.bouncer.enabled')) {
+            Bouncer::get($status);
+        }
+
+        Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
+        $hideNsfw = config('instance.hide_nsfw_on_public_feeds');
+        if( $status->uri == null &&
+            $status->scope == 'public' &&
+            in_array($status->type, $types) &&
+            $status->in_reply_to_id === null &&
+            $status->reblog_of_id === null &&
+            ($hideNsfw ? $status->is_nsfw == false : true)
+        ) {
+            if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
+                PublicTimelineService::add($status->id);
+            }
+        }
+
+        if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
+            StatusActivityPubDeliver::dispatch($status);
+        }
+    }
 }
--- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php
+++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php
@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;

 class StatusTagsPipeline implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

-	protected $activity;
-	protected $status;
+    protected $activity;
+    protected $status;

-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct($activity, $status)
-	{
-		$this->activity = $activity;
-		$this->status = $status;
-	}
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($activity, $status)
+    {
+        $this->activity = $activity;
+        $this->status = $status;
+    }

-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$res = $this->activity;
-		$status = $this->status;
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $res = $this->activity;
+        $status = $this->status;

        if(isset($res['tag']['type'], $res['tag']['name'])) {
            $res['tag'] = [$res['tag']];
        }

-		$tags = collect($res['tag']);
+        $tags = collect($res['tag']);

-		// Emoji
-		$tags->filter(function($tag) {
-			return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
-		})
-		->map(function($tag) {
-			CustomEmojiService::import($tag['id'], $this->status->id);
-		});
+        // Emoji
+        $tags->filter(function($tag) {
+            return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
+        })
+        ->map(function($tag) {
+            CustomEmojiService::import($tag['id'], $this->status->id);
+        });

-		// Hashtags
-		$tags->filter(function($tag) {
-			return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
-		})
-		->map(function($tag) use($status) {
-			$name = substr($tag['name'], 0, 1) == '#' ?
-				substr($tag['name'], 1) : $tag['name'];
+        // Hashtags
+        $tags->filter(function($tag) {
+            return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
+        })
+        ->map(function($tag) use($status) {
+            $name = substr($tag['name'], 0, 1) == '#' ?
+                substr($tag['name'], 1) : $tag['name'];

-			$banned = TrendingHashtagService::getBannedHashtagNames();
+            $banned = TrendingHashtagService::getBannedHashtagNames();

-			if(count($banned)) {
+            if(count($banned)) {
                if(in_array(strtolower($name), array_map('strtolower', $banned))) {
-                   	return;
+                    return;
                }
            }

            if(config('database.default') === 'pgsql') {
-            	$hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name, '-', false))
-            		->first();
+                $hashtag = Hashtag::where('name', 'ilike', $name)
+                    ->orWhere('slug', 'ilike', str_slug($name, '-', false))
+                    ->first();

-				if(!$hashtag) {
-					$hashtag = Hashtag::updateOrCreate([
-						'slug' => str_slug($name, '-', false),
-						'name' => $name
-					]);
-				}
+                if(!$hashtag) {
+                    $hashtag = Hashtag::updateOrCreate([
+                        'slug' => str_slug($name, '-', false),
+                        'name' => $name
+                    ]);
+                }
            } else {
-				$hashtag = Hashtag::updateOrCreate([
-					'slug' => str_slug($name, '-', false),
-					'name' => $name
-				]);
+                $hashtag = Hashtag::updateOrCreate([
+                    'slug' => str_slug($name, '-', false),
+                    'name' => $name
+                ]);
            }

-			StatusHashtag::firstOrCreate([
-				'status_id' => $status->id,
-				'hashtag_id' => $hashtag->id,
-				'profile_id' => $status->profile_id,
-				'status_visibility' => $status->scope
-			]);
-		});
+            StatusHashtag::firstOrCreate([
+                'status_id' => $status->id,
+                'hashtag_id' => $hashtag->id,
+                'profile_id' => $status->profile_id,
+                'status_visibility' => $status->scope
+            ]);
+        });

-		// Mentions
-		$tags->filter(function($tag) {
-			return $tag &&
-				$tag['type'] == 'Mention' &&
-				isset($tag['href']) &&
-				substr($tag['href'], 0, 8) === 'https://';
-		})
-		->map(function($tag) use($status) {
-			if(Helpers::validateLocalUrl($tag['href'])) {
-				$parts = explode('/', $tag['href']);
-				if(!$parts) {
-					return;
-				}
-				$pid = AccountService::usernameToId(end($parts));
-				if(!$pid) {
-					return;
-				}
-			} else {
-				$acct = Helpers::profileFetch($tag['href']);
-				if(!$acct) {
-					return;
-				}
-				$pid = $acct->id;
-			}
-			$mention = new Mention;
-			$mention->status_id = $status->id;
-			$mention->profile_id = $pid;
-			$mention->save();
-			MentionPipeline::dispatch($status, $mention);
-		});
-	}
+        // Mentions
+        $tags->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Mention' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        })
+        ->map(function($tag) use($status) {
+            if(Helpers::validateLocalUrl($tag['href'])) {
+                $parts = explode('/', $tag['href']);
+                if(!$parts) {
+                    return;
+                }
+                $pid = AccountService::usernameToId(end($parts));
+                if(!$pid) {
+                    return;
+                }
+            } else {
+                $acct = Helpers::profileFetch($tag['href']);
+                if(!$acct) {
+                    return;
+                }
+                $pid = $acct->id;
+            }
+            $mention = new Mention;
+            $mention->status_id = $status->id;
+            $mention->profile_id = $pid;
+            $mention->save();
+            MentionPipeline::dispatch($status, $mention);
+        });
+
+        StatusService::refresh($status->id);
+    }
 }
--- a/app/Services/StatusHashtagService.php
+++ b/app/Services/StatusHashtagService.php
@ -84,18 +84,14 @@ class StatusHashtagService {

 	public static function statusTags($statusId)
 	{
-		$key = 'pf:services:sh:id:' . $statusId;
+		$status = Status::with('hashtags')->find($statusId);
+		if(!$status) {
+			return [];
+		}

-		return Cache::remember($key, 604800, function() use($statusId) {
-			$status = Status::find($statusId);
-			if(!$status) {
-				return [];
-			}
-
-			$fractal = new Fractal\Manager();
-			$fractal->setSerializer(new ArraySerializer());
-			$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
-			return $fractal->createData($resource)->toArray();
-		});
+		$fractal = new Fractal\Manager();
+		$fractal->setSerializer(new ArraySerializer());
+		$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
+		return $fractal->createData($resource)->toArray();
 	}
 }