Merge pull request #4777 from pixelfed/staging

Add Related Hashtags

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
 - Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
 - Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
+- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
 
 ### Federation
 - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@@ -58,6 +59,8 @@
 - Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
 - Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
 - Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
+- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/HashtagRelatedGenerate.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+use App\Services\HashtagRelatedService;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use function Laravel\Prompts\multiselect;
+
+class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-related-generate {tag}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'tag' => 'Which hashtag should we generate related tags for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $tag = $this->argument('tag');
+        $hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
+        if(!$hashtag) {
+            $this->error('Hashtag not found, aborting...');
+            exit;
+        }
+
+        $this->info('Looking up #' . $tag . '...');
+
+        $tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
+        if(!$tags || $tags < 100) {
+            $this->error('Not enough posts found to generate related hashtags!');
+            exit;
+        }
+
+        $this->info('Found ' . $tags . ' posts that use that hashtag');
+        $related = collect(HashtagRelatedService::fetchRelatedTags($tag));
+
+        $selected = multiselect(
+            label: 'Which tags do you want to generate?',
+            options: $related->pluck('name'),
+            required: true,
+        );
+
+        $filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
+        $agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
+
+        HashtagRelated::updateOrCreate([
+            'hashtag_id' => $hashtag->id,
+        ], [
+            'related_tags' => array_values($filtered),
+            'agg_score' => $agg_score,
+            'last_calculated_at' => now()
+        ]);
+
+        $this->info('Finished!');
+    }
+}

app/Http/Controllers/Api/V1/TagsController.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Hashtag;
+use App\HashtagFollow;
+use App\StatusHashtag;
+use App\Services\AccountService;
+use App\Services\HashtagService;
+use App\Services\HashtagFollowService;
+use App\Services\HashtagRelatedService;
+use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+
+class TagsController extends Controller
+{
+    const PF_API_ENTITY_KEY = "_pe";
+
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    /**
+    * GET /api/v1/tags/:id/related
+    *
+    *
+    * @return array
+    */
+    public function relatedTags(Request $request, $tag)
+    {
+        abort_unless($request->user(), 403);
+        $tag = Hashtag::whereSlug($tag)->firstOrFail();
+        return HashtagRelatedService::get($tag->id);
+    }
+
+    /**
+    * POST /api/v1/tags/:id/follow
+    *
+    *
+    * @return object
+    */
+    public function followHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        abort_if(
+            HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
+            422,
+            'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
+        );
+
+        $follows = HashtagFollow::updateOrCreate(
+            [
+                'profile_id' => $account['id'],
+                'hashtag_id' => $tag->id
+            ],
+            [
+                'user_id' => $request->user()->id
+            ]
+        );
+
+        HashtagService::follow($pid, $tag->id);
+        HashtagFollowService::add($tag->id, $pid);
+
+        return response()->json(FollowedTagResource::make($follows)->toArray($request));
+    }
+
+    /**
+    * POST /api/v1/tags/:id/unfollow
+    *
+    *
+    * @return object
+    */
+    public function unfollowHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        $follows = HashtagFollow::whereProfileId($pid)
+            ->whereHashtagId($tag->id)
+            ->first();
+
+        if(!$follows) {
+            return [
+                'name' => $tag->name,
+                'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        if($follows) {
+            HashtagService::unfollow($pid, $tag->id);
+            HashtagFollowService::unfollow($tag->id, $pid);
+            HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
+            $follows->delete();
+        }
+
+        $res = FollowedTagResource::make($follows)->toArray($request);
+        $res['following'] = false;
+        return response()->json($res);
+    }
+
+    /**
+    * GET /api/v1/tags/:id
+    *
+    *
+    * @return object
+    */
+    public function getHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        if(!$tag) {
+            return [
+                'name' => $id,
+                'url' => config('app.url') . '/i/web/hashtag/' . $id,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        $res = [
+            'name' => $tag->name,
+            'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+            'history' => [],
+            'following' => HashtagService::isFollowing($pid, $tag->id)
+        ];
+
+        if($request->has(self::PF_API_ENTITY_KEY)) {
+            $res['count'] = HashtagService::count($tag->id);
+        }
+
+        return $this->json($res);
+    }
+
+    /**
+    * GET /api/v1/followed_tags
+    *
+    *
+    * @return array
+    */
+    public function getFollowedTags(Request $request)
+    {
+        abort_if(!$request->user(), 403);
+
+        $account = AccountService::get($request->user()->profile_id);
+
+        $this->validate($request, [
+            'cursor' => 'sometimes',
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+
+        $res = HashtagFollow::whereProfileId($account['id'])
+            ->orderByDesc('id')
+            ->cursorPaginate($limit)
+            ->withQueryString();
+
+        $pagination = false;
+        $prevPage = $res->nextPageUrl();
+        $nextPage = $res->previousPageUrl();
+        if($nextPage && $prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
+        } else if($nextPage && !$prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next"';
+        } else if(!$nextPage && $prevPage) {
+            $pagination = '<' . $prevPage . '>; rel="prev"';
+        }
+
+        if($pagination) {
+            return response()->json(FollowedTagResource::collection($res)->collection)
+                ->header('Link', $pagination);
+        }
+        return response()->json(FollowedTagResource::collection($res)->collection);
+    }
+}

app/Models/HashtagRelated.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HashtagRelated extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    /**
+     * The attributes that should be mutated to dates and other custom formats.
+     *
+     * @var array
+     */
+    protected $casts = [
+        'related_tags' => 'array',
+        'last_calculated_at' => 'datetime',
+        'last_moderated_at' => 'datetime',
+    ];
+}

app/Services/HashtagRelatedService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use DB;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+
+class HashtagRelatedService
+{
+    public static function get($id)
+    {
+        $tag = HashtagRelated::whereHashtagId($id)->first();
+        if(!$tag) {
+            return [];
+        }
+        return $tag->related_tags;
+    }
+
+    public static function fetchRelatedTags($tag)
+    {
+        $res = StatusHashtag::query()
+            ->select('h2.name', DB::raw('COUNT(*) as related_count'))
+            ->join('status_hashtags as hs2', function ($join) {
+                $join->on('status_hashtags.status_id', '=', 'hs2.status_id')
+                     ->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
+            })
+            ->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
+            ->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
+            ->where('h1.name', '=', $tag)
+            ->groupBy('h2.name')
+            ->orderBy('related_count', 'desc')
+            ->limit(30)
+            ->get();
+
+        return $res;
+    }
+}

database/migrations/2023_11_16_124107_create_hashtag_related_table.php

@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('hashtag_related', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->bigInteger('hashtag_id')->unsigned()->unique()->index();
+            $table->json('related_tags')->nullable();
+            $table->bigInteger('agg_score')->unsigned()->nullable()->index();
+            $table->timestamp('last_calculated_at')->nullable()->index();
+            $table->timestamp('last_moderated_at')->nullable()->index();
+            $table->boolean('skip_refresh')->default(false)->index();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('hashtag_related');
+    }
+};

routes/api.php

@@ -2,6 +2,7 @@
 
 use Illuminate\Http\Request;
 use App\Http\Middleware\DeprecatedEndpoint;
+use App\Http\Controllers\Api\V1\TagsController;
 
 $middleware = ['auth:api','validemail'];
 
@@ -92,10 +93,11 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
         Route::get('markers', 'Api\ApiV1Controller@getMarkers')->middleware($middleware);
         Route::post('markers', 'Api\ApiV1Controller@setMarkers')->middleware($middleware);
 
-        Route::get('followed_tags', 'Api\ApiV1Controller@getFollowedTags')->middleware($middleware);
+        Route::get('followed_tags', [TagsController::class, 'getFollowedTags'])->middleware($middleware);
-        Route::post('tags/{id}/follow', 'Api\ApiV1Controller@followHashtag')->middleware($middleware);
+        Route::post('tags/{id}/follow', [TagsController::class, 'followHashtag'])->middleware($middleware);
-        Route::post('tags/{id}/unfollow', 'Api\ApiV1Controller@unfollowHashtag')->middleware($middleware);
+        Route::post('tags/{id}/unfollow', [TagsController::class, 'unfollowHashtag'])->middleware($middleware);
-        Route::get('tags/{id}', 'Api\ApiV1Controller@getHashtag')->middleware($middleware);
+        Route::get('tags/{id}/related', [TagsController::class, 'relatedTags'])->middleware($middleware);
+        Route::get('tags/{id}', [TagsController::class, 'getHashtag'])->middleware($middleware);
 
         Route::get('statuses/{id}/history', 'StatusEditController@history')->middleware($middleware);
         Route::put('statuses/{id}', 'StatusEditController@store')->middleware($middleware);