Merge pull request #4777 from pixelfed/staging

Add Related Hashtags

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
 - Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
 - Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
+- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
 
 ### Federation
 - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@@ -58,6 +59,8 @@
 - Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
 - Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
 - Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
+- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/HashtagRelatedGenerate.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+use App\Services\HashtagRelatedService;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use function Laravel\Prompts\multiselect;
+
+class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-related-generate {tag}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'tag' => 'Which hashtag should we generate related tags for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $tag = $this->argument('tag');
+        $hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
+        if(!$hashtag) {
+            $this->error('Hashtag not found, aborting...');
+            exit;
+        }
+
+        $this->info('Looking up #' . $tag . '...');
+
+        $tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
+        if(!$tags || $tags < 100) {
+            $this->error('Not enough posts found to generate related hashtags!');
+            exit;
+        }
+
+        $this->info('Found ' . $tags . ' posts that use that hashtag');
+        $related = collect(HashtagRelatedService::fetchRelatedTags($tag));
+
+        $selected = multiselect(
+            label: 'Which tags do you want to generate?',
+            options: $related->pluck('name'),
+            required: true,
+        );
+
+        $filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
+        $agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
+
+        HashtagRelated::updateOrCreate([
+            'hashtag_id' => $hashtag->id,
+        ], [
+            'related_tags' => array_values($filtered),
+            'agg_score' => $agg_score,
+            'last_calculated_at' => now()
+        ]);
+
+        $this->info('Finished!');
+    }
+}

app/Http/Controllers/Api/ApiV1Controller.php

@@ -2146,7 +2146,7 @@ class ApiV1Controller extends Controller
         $max = $request->input('max_id');
         $limit = $request->input('limit') ?? 20;
         $pid = $request->user()->profile_id;
-		$includeReblogs = $request->filled('include_reblogs');
+        $includeReblogs = $request->filled('include_reblogs') ? $request->boolean('include_reblogs') : false;
         $nullFields = $includeReblogs ?
         ['in_reply_to_id'] :
         ['in_reply_to_id', 'reblog_of_id'];
@@ -2155,14 +2155,15 @@ class ApiV1Controller extends Controller
         ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'];
 
         if(config('exp.cached_home_timeline')) {
+            $paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50;
             if($min || $max) {
                 if($request->has('min_id')) {
-					$res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $limit + 10);
+                    $res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
                 } else {
-					$res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $limit + 10);
+                    $res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
                 }
             } else {
-				$res = HomeTimelineService::get($pid, 0, $limit + 10);
+                $res = HomeTimelineService::get($pid, 0, $paddedLimit);
             }
 
             if(!$res) {
@@ -3712,171 +3713,4 @@ class ApiV1Controller extends Controller
 
         return $this->json([]);
     }
-
-	/**
-	* GET /api/v1/followed_tags
-	*
-	*
-	* @return array
-	*/
-	public function getFollowedTags(Request $request)
-	{
-		abort_if(!$request->user(), 403);
-
-		$account = AccountService::get($request->user()->profile_id);
-
-		$this->validate($request, [
-			'cursor' => 'sometimes',
-			'limit' => 'sometimes|integer|min:1|max:200'
-		]);
-		$limit = $request->input('limit', 100);
-
-		$res = HashtagFollow::whereProfileId($account['id'])
-			->orderByDesc('id')
-			->cursorPaginate($limit)->withQueryString();
-
-		$pagination = false;
-		$prevPage = $res->nextPageUrl();
-		$nextPage = $res->previousPageUrl();
-		if($nextPage && $prevPage) {
-			$pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
-		} else if($nextPage && !$prevPage) {
-			$pagination = '<' . $nextPage . '>; rel="next"';
-		} else if(!$nextPage && $prevPage) {
-			$pagination = '<' . $prevPage . '>; rel="prev"';
-		}
-
-		if($pagination) {
-			return response()->json(FollowedTagResource::collection($res)->collection)
-				->header('Link', $pagination);
-		}
-		return response()->json(FollowedTagResource::collection($res)->collection);
-	}
-
-	/**
-	* POST /api/v1/tags/:id/follow
-	*
-	*
-	* @return object
-	*/
-	public function followHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		abort_if(!$tag, 422, 'Unknown hashtag');
-
-		abort_if(
-			HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
-			422,
-			'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
-		);
-
-		$follows = HashtagFollow::updateOrCreate(
-			[
-				'profile_id' => $account['id'],
-				'hashtag_id' => $tag->id
-			],
-			[
-				'user_id' => $request->user()->id
-			]
-		);
-
-		HashtagService::follow($pid, $tag->id);
-		HashtagFollowService::add($tag->id, $pid);
-
-		return response()->json(FollowedTagResource::make($follows)->toArray($request));
-	}
-
-	/**
-	* POST /api/v1/tags/:id/unfollow
-	*
-	*
-	* @return object
-	*/
-	public function unfollowHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		abort_if(!$tag, 422, 'Unknown hashtag');
-
-		$follows = HashtagFollow::whereProfileId($pid)
-			->whereHashtagId($tag->id)
-			->first();
-
-		if(!$follows) {
-			return [
-				'name' => $tag->name,
-				'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
-				'history' => [],
-				'following' => false
-			];
-		}
-
-		if($follows) {
-			HashtagService::unfollow($pid, $tag->id);
-			HashtagFollowService::unfollow($tag->id, $pid);
-			HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
-			$follows->delete();
-		}
-
-		$res = FollowedTagResource::make($follows)->toArray($request);
-		$res['following'] = false;
-		return response()->json($res);
-	}
-
-	/**
-	* GET /api/v1/tags/:id
-	*
-	*
-	* @return object
-	*/
-	public function getHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		if(!$tag) {
-			return [
-				'name' => $id,
-				'url' => config('app.url') . '/i/web/hashtag/' . $id,
-				'history' => [],
-				'following' => false
-			];
-		}
-
-		$res = [
-			'name' => $tag->name,
-			'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
-			'history' => [],
-			'following' => HashtagService::isFollowing($pid, $tag->id)
-		];
-
-		if($request->has(self::PF_API_ENTITY_KEY)) {
-			$res['count'] = HashtagService::count($tag->id);
-		}
-
-		return $this->json($res);
-	}
 }

app/Http/Controllers/Api/V1/TagsController.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Hashtag;
+use App\HashtagFollow;
+use App\StatusHashtag;
+use App\Services\AccountService;
+use App\Services\HashtagService;
+use App\Services\HashtagFollowService;
+use App\Services\HashtagRelatedService;
+use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+
+class TagsController extends Controller
+{
+    const PF_API_ENTITY_KEY = "_pe";
+
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    /**
+    * GET /api/v1/tags/:id/related
+    *
+    *
+    * @return array
+    */
+    public function relatedTags(Request $request, $tag)
+    {
+        abort_unless($request->user(), 403);
+        $tag = Hashtag::whereSlug($tag)->firstOrFail();
+        return HashtagRelatedService::get($tag->id);
+    }
+
+    /**
+    * POST /api/v1/tags/:id/follow
+    *
+    *
+    * @return object
+    */
+    public function followHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        abort_if(
+            HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
+            422,
+            'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
+        );
+
+        $follows = HashtagFollow::updateOrCreate(
+            [
+                'profile_id' => $account['id'],
+                'hashtag_id' => $tag->id
+            ],
+            [
+                'user_id' => $request->user()->id
+            ]
+        );
+
+        HashtagService::follow($pid, $tag->id);
+        HashtagFollowService::add($tag->id, $pid);
+
+        return response()->json(FollowedTagResource::make($follows)->toArray($request));
+    }
+
+    /**
+    * POST /api/v1/tags/:id/unfollow
+    *
+    *
+    * @return object
+    */
+    public function unfollowHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        $follows = HashtagFollow::whereProfileId($pid)
+            ->whereHashtagId($tag->id)
+            ->first();
+
+        if(!$follows) {
+            return [
+                'name' => $tag->name,
+                'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        if($follows) {
+            HashtagService::unfollow($pid, $tag->id);
+            HashtagFollowService::unfollow($tag->id, $pid);
+            HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
+            $follows->delete();
+        }
+
+        $res = FollowedTagResource::make($follows)->toArray($request);
+        $res['following'] = false;
+        return response()->json($res);
+    }
+
+    /**
+    * GET /api/v1/tags/:id
+    *
+    *
+    * @return object
+    */
+    public function getHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        if(!$tag) {
+            return [
+                'name' => $id,
+                'url' => config('app.url') . '/i/web/hashtag/' . $id,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        $res = [
+            'name' => $tag->name,
+            'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+            'history' => [],
+            'following' => HashtagService::isFollowing($pid, $tag->id)
+        ];
+
+        if($request->has(self::PF_API_ENTITY_KEY)) {
+            $res['count'] = HashtagService::count($tag->id);
+        }
+
+        return $this->json($res);
+    }
+
+    /**
+    * GET /api/v1/followed_tags
+    *
+    *
+    * @return array
+    */
+    public function getFollowedTags(Request $request)
+    {
+        abort_if(!$request->user(), 403);
+
+        $account = AccountService::get($request->user()->profile_id);
+
+        $this->validate($request, [
+            'cursor' => 'sometimes',
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+
+        $res = HashtagFollow::whereProfileId($account['id'])
+            ->orderByDesc('id')
+            ->cursorPaginate($limit)
+            ->withQueryString();
+
+        $pagination = false;
+        $prevPage = $res->nextPageUrl();
+        $nextPage = $res->previousPageUrl();
+        if($nextPage && $prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
+        } else if($nextPage && !$prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next"';
+        } else if(!$nextPage && $prevPage) {
+            $pagination = '<' . $prevPage . '>; rel="prev"';
+        }
+
+        if($pagination) {
+            return response()->json(FollowedTagResource::collection($res)->collection)
+                ->header('Link', $pagination);
+        }
+        return response()->json(FollowedTagResource::collection($res)->collection);
+    }
+}

app/Models/HashtagRelated.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HashtagRelated extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    /**
+     * The attributes that should be mutated to dates and other custom formats.
+     *
+     * @var array
+     */
+    protected $casts = [
+        'related_tags' => 'array',
+        'last_calculated_at' => 'datetime',
+        'last_moderated_at' => 'datetime',
+    ];
+}

app/Services/HashtagRelatedService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use DB;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+
+class HashtagRelatedService
+{
+    public static function get($id)
+    {
+        $tag = HashtagRelated::whereHashtagId($id)->first();
+        if(!$tag) {
+            return [];
+        }
+        return $tag->related_tags;
+    }
+
+    public static function fetchRelatedTags($tag)
+    {
+        $res = StatusHashtag::query()
+            ->select('h2.name', DB::raw('COUNT(*) as related_count'))
+            ->join('status_hashtags as hs2', function ($join) {
+                $join->on('status_hashtags.status_id', '=', 'hs2.status_id')
+                     ->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
+            })
+            ->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
+            ->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
+            ->where('h1.name', '=', $tag)
+            ->groupBy('h2.name')
+            ->orderBy('related_count', 'desc')
+            ->limit(30)
+            ->get();
+
+        return $res;
+    }
+}

database/migrations/2023_11_16_124107_create_hashtag_related_table.php

@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('hashtag_related', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->bigInteger('hashtag_id')->unsigned()->unique()->index();
+            $table->json('related_tags')->nullable();
+            $table->bigInteger('agg_score')->unsigned()->nullable()->index();
+            $table->timestamp('last_calculated_at')->nullable()->index();
+            $table->timestamp('last_moderated_at')->nullable()->index();
+            $table->boolean('skip_refresh')->default(false)->index();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('hashtag_related');
+    }
+};

routes/api.php

@@ -2,6 +2,7 @@
 
 use Illuminate\Http\Request;
 use App\Http\Middleware\DeprecatedEndpoint;
+use App\Http\Controllers\Api\V1\TagsController;
 
 $middleware = ['auth:api','validemail'];
 
@@ -92,10 +93,11 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
         Route::get('markers', 'Api\ApiV1Controller@getMarkers')->middleware($middleware);
         Route::post('markers', 'Api\ApiV1Controller@setMarkers')->middleware($middleware);
 
-        Route::get('followed_tags', 'Api\ApiV1Controller@getFollowedTags')->middleware($middleware);
+        Route::get('followed_tags', [TagsController::class, 'getFollowedTags'])->middleware($middleware);
-        Route::post('tags/{id}/follow', 'Api\ApiV1Controller@followHashtag')->middleware($middleware);
+        Route::post('tags/{id}/follow', [TagsController::class, 'followHashtag'])->middleware($middleware);
-        Route::post('tags/{id}/unfollow', 'Api\ApiV1Controller@unfollowHashtag')->middleware($middleware);
+        Route::post('tags/{id}/unfollow', [TagsController::class, 'unfollowHashtag'])->middleware($middleware);
-        Route::get('tags/{id}', 'Api\ApiV1Controller@getHashtag')->middleware($middleware);
+        Route::get('tags/{id}/related', [TagsController::class, 'relatedTags'])->middleware($middleware);
+        Route::get('tags/{id}', [TagsController::class, 'getHashtag'])->middleware($middleware);
 
         Route::get('statuses/{id}/history', 'StatusEditController@history')->middleware($middleware);
         Route::put('statuses/{id}', 'StatusEditController@store')->middleware($middleware);