merge dev

.editorconfig

@@ -1,8 +1,8 @@
 root = true
 
 [*]
+indent_style = space
 indent_size = 4
-indent_style = tab
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true

CHANGELOG.md

@@ -1,6 +1,89 @@
 # Release Notes
 
 ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
+
+### Added
+- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
+- Video WebP2P ([#4713](https://github.com/pixelfed/pixelfed/pull/4713)) ([0405ef12](https://github.com/pixelfed/pixelfed/commit/0405ef12))
+- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
+- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
+- Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
+- Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
+- Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
+- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
+- Added Mutual Followers API endpoint ([33dbbe46](https://github.com/pixelfed/pixelfed/commit/33dbbe46))
+- Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac))
+
+### Federation
+- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
+- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
+
+### Updates
+- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59))
+- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904))
+- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce))
+- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d))
+- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec))
+- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d))
+- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973))
+- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7))
+- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605))
+- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27))
+- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
+- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
+- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
+- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
+- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
+- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
+- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
+- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
+- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
+- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed))
+- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3))
+- Update StatusTransformer, generate autolink on request ([dfe2379b](https://github.com/pixelfed/pixelfed/commit/dfe2379b))
+- Update ComposeModal component, fix multi filter bug and allow media re-ordering before upload/posting ([56e315f6](https://github.com/pixelfed/pixelfed/commit/56e315f6))
+- Update ApiV1Dot1Controller, allow iar rate limits to be configurable ([28a80803](https://github.com/pixelfed/pixelfed/commit/28a80803))
+- Update ApiV1Dot1Controller, add domain to iar redirect ([1f82d47c](https://github.com/pixelfed/pixelfed/commit/1f82d47c))
+- Update ApiV1Dot1Controller, add configurable app confirm rate limit ttl ([4c6a0719](https://github.com/pixelfed/pixelfed/commit/4c6a0719))
+- Update LikePipeline, dispatch to feed queue. Fixes ([#4723](https://github.com/pixelfed/pixelfed/issues/4723)) ([da510089](https://github.com/pixelfed/pixelfed/commit/da510089))
+- Update AccountImport ([5a2d7e3e](https://github.com/pixelfed/pixelfed/commit/5a2d7e3e))
+- Update ImportPostController, fix IG bug with missing spaces between hashtags ([9c24157a](https://github.com/pixelfed/pixelfed/commit/9c24157a))
+- Update ApiV1Controller, fix mutes in home feed ([ddc21714](https://github.com/pixelfed/pixelfed/commit/ddc21714))
+- Update AP helpers, improve preferredUsername validation ([21218c79](https://github.com/pixelfed/pixelfed/commit/21218c79))
+- Update delete pipelines, properly invoke StatusHashtag delete events ([ce54d29c](https://github.com/pixelfed/pixelfed/commit/ce54d29c))
+- Update mail config ([0e431271](https://github.com/pixelfed/pixelfed/commit/0e431271))
+- Update hashtag following ([015b1b80](https://github.com/pixelfed/pixelfed/commit/015b1b80))
+- Update IncrementPostCount job, prevent overlap ([b2c9cc23](https://github.com/pixelfed/pixelfed/commit/b2c9cc23))
+- Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885))
+- Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae))
+- Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008))
+- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
+- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
+- Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
+- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
+- Update StoryApiV1Controller, add self-carousel endpoint. Fixes ([#4352](https://github.com/pixelfed/pixelfed/issues/4352)) ([bcb88d5b](https://github.com/pixelfed/pixelfed/commit/bcb88d5b))
+- Update FollowServiceWarmCache, use more efficient query ([fe9b4c5a](https://github.com/pixelfed/pixelfed/commit/fe9b4c5a))
+- Update HomeFeedPipeline, observe mutes/blocks during fanout ([8548294c](https://github.com/pixelfed/pixelfed/commit/8548294c))
+- Update FederationController, add proper following/follower counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
+- Update FederationController, add proper statuses counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
+- Update Inbox handler, fix missing object_url and uri fields for direct statuses ([a0157fce](https://github.com/pixelfed/pixelfed/commit/a0157fce))
+- Update DirectMessageController, deliver direct delete activities to user inbox instead of sharedInbox ([d848792a](https://github.com/pixelfed/pixelfed/commit/d848792a))
+- Update DirectMessageController, dispatch deliver and delete actions to the job queue ([7f462a80](https://github.com/pixelfed/pixelfed/commit/7f462a80))
+- Update Inbox, improve story attribute collection ([06bee36c](https://github.com/pixelfed/pixelfed/commit/06bee36c))
+- Update DirectMessageController, dispatch local deletes to pipeline ([98186564](https://github.com/pixelfed/pixelfed/commit/98186564))
+- Update StatusPipeline, fix Direct and Story notification deletion ([4c95306f](https://github.com/pixelfed/pixelfed/commit/4c95306f))
+- Update Notifications.vue, fix deprecated DM action links for story activities ([4c3823b0](https://github.com/pixelfed/pixelfed/commit/4c3823b0))
+- Update ComposeModal, fix missing alttext post state ([0a068119](https://github.com/pixelfed/pixelfed/commit/0a068119))
+- Update PhotoAlbumPresenter.vue, fix fullscreen mode ([822e9888](https://github.com/pixelfed/pixelfed/commit/822e9888))
+- Update Timeline.vue, improve CHT pagination ([9c43e7e2](https://github.com/pixelfed/pixelfed/commit/9c43e7e2))
+- Update HomeFeedPipeline, fix StatusService validation ([041c0135](https://github.com/pixelfed/pixelfed/commit/041c0135))
+- Update Inbox, improve tombstone query efficiency ([759a4393](https://github.com/pixelfed/pixelfed/commit/759a4393))
+- Update AccountService, add setLastActive method ([ebbd98e7](https://github.com/pixelfed/pixelfed/commit/ebbd98e7))
+- Update ApiV1Controller, set last_active_at ([b6419545](https://github.com/pixelfed/pixelfed/commit/b6419545))
+- Update AdminShadowFilter, fix deleted profile bug ([a492a95a](https://github.com/pixelfed/pixelfed/commit/a492a95a))
+- Update FollowerService, add $silent param to remove method to more efficently purge relationships ([1664a5bc](https://github.com/pixelfed/pixelfed/commit/1664a5bc))
+- Update AP ProfileTransformer, add published attribute ([adfaa2b1](https://github.com/pixelfed/pixelfed/commit/adfaa2b1))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/AddUserDomainBlock.php

@@ -0,0 +1,106 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\User;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
+use function Laravel\Prompts\text;
+use function Laravel\Prompts\confirm;
+use function Laravel\Prompts\progress;
+
+class AddUserDomainBlock extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:add-user-domain-block';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Apply a domain block to all users';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $domain = text('Enter domain you want to block');
+        $domain = strtolower($domain);
+        $domain = $this->validateDomain($domain);
+        if(!$domain || empty($domain)) {
+            $this->error('Invalid domain');
+            return;
+        }
+        $this->processBlocks($domain);
+        return;
+    }
+
+    protected function validateDomain($domain)
+    {
+        if(!strpos($domain, '.')) {
+            return;
+        }
+
+        if(str_starts_with($domain, 'https://')) {
+            $domain = str_replace('https://', '', $domain);
+        }
+
+        if(str_starts_with($domain, 'http://')) {
+            $domain = str_replace('http://', '', $domain);
+        }
+
+        $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
+
+        $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
+        if(!$valid) {
+            return;
+        }
+
+        if($domain === config('pixelfed.domain.app')) {
+            $this->error('Invalid domain');
+            return;
+        }
+
+        $confirmed = confirm('Are you sure you want to block ' . $domain . '?');
+        if(!$confirmed) {
+            return;
+        }
+
+        return $domain;
+    }
+
+    protected function processBlocks($domain)
+    {
+        DefaultDomainBlock::updateOrCreate([
+            'domain' => $domain
+        ]);
+        progress(
+            label: 'Updating user domain blocks...',
+            steps: User::lazyById(500),
+            callback: fn ($user) => $this->performTask($user, $domain),
+        );
+    }
+
+    protected function performTask($user, $domain)
+    {
+        if(!$user->profile_id || $user->delete_after) {
+            return;
+        }
+
+        if($user->status != null && $user->status != 'disabled') {
+            return;
+        }
+
+        UserDomainBlock::updateOrCreate([
+            'profile_id' => $user->profile_id,
+            'domain' => $domain
+        ]);
+    }
+}

app/Console/Commands/AvatarStorageDeepClean.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Cache;
+use Storage;
+use App\Avatar;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
+
+class AvatarStorageDeepClean extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'avatar:storage-deep-clean';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Cleanup avatar storage';
+
+    protected $shouldKeepRunning = true;
+    protected $counter = 0;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Pixelfed Avatar Deep Cleaner');
+        $this->line(' ');
+        $this->info('    Purge/delete old and outdated avatars from remote accounts');
+        $this->line(' ');
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
+            exit;
+        }
+
+        $start = 0;
+
+        if(!$this->confirm('Are you sure you want to proceed?')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        if(!$this->activeCheck()) {
+            $this->info('Found existing deep cleaning job');
+            if(!$this->confirm('Do you want to continue where you left off?')) {
+                $this->error('Aborting...');
+                exit;
+            } else {
+                $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
+
+                if($start && $start < 1 || $start > PHP_INT_MAX) {
+                    $this->error('Error fetching cached value');
+                    $this->error('Aborting...');
+                    exit;
+                }
+            }
+        }
+
+        $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
+        $bar = $this->output->createProgressBar($count);
+
+        foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
+            usleep(random_int(50, 1000));
+            $this->counter++;
+            $this->handleAvatar($avatar);
+            $bar->advance();
+        }
+        $bar->finish();
+    }
+
+    protected function updateCache($id)
+    {
+        Cache::put('cmd:asdp', $id);
+        if($this->counter % 5 === 0) {
+            Storage::put('avatar-deep-clean.json', $id);
+        }
+    }
+
+    protected function activeCheck()
+    {
+        if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    protected function handleAvatar($avatar)
+    {
+        $this->updateCache($avatar->id);
+        $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
+        $queue = $queues[random_int(0, 7)];
+        AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
+    }
+}

app/Console/Commands/DeleteUserDomainBlock.php

@@ -0,0 +1,96 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\User;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
+use function Laravel\Prompts\text;
+use function Laravel\Prompts\confirm;
+use function Laravel\Prompts\progress;
+
+class DeleteUserDomainBlock extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:delete-user-domain-block';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Remove a domain block for all users';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $domain = text('Enter domain you want to unblock');
+        $domain = strtolower($domain);
+        $domain = $this->validateDomain($domain);
+        if(!$domain || empty($domain)) {
+            $this->error('Invalid domain');
+            return;
+        }
+        $this->processUnblocks($domain);
+        return;
+    }
+
+    protected function validateDomain($domain)
+    {
+        if(!strpos($domain, '.')) {
+            return;
+        }
+
+        if(str_starts_with($domain, 'https://')) {
+            $domain = str_replace('https://', '', $domain);
+        }
+
+        if(str_starts_with($domain, 'http://')) {
+            $domain = str_replace('http://', '', $domain);
+        }
+
+        $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
+
+        $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
+        if(!$valid) {
+            return;
+        }
+
+        if($domain === config('pixelfed.domain.app')) {
+            return;
+        }
+
+        $confirmed = confirm('Are you sure you want to unblock ' . $domain . '?');
+        if(!$confirmed) {
+            return;
+        }
+
+        return $domain;
+    }
+
+    protected function processUnblocks($domain)
+    {
+        DefaultDomainBlock::whereDomain($domain)->delete();
+        if(!UserDomainBlock::whereDomain($domain)->count()) {
+            $this->info('No results found!');
+            return;
+        }
+        progress(
+            label: 'Updating user domain blocks...',
+            steps: UserDomainBlock::whereDomain($domain)->lazyById(500),
+            callback: fn ($domainBlock) => $this->performTask($domainBlock),
+        );
+    }
+
+    protected function performTask($domainBlock)
+    {
+        $domainBlock->deleteQuietly();
+    }
+}

app/Console/Commands/HashtagCachedCountUpdate.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use DB;
+
+class HashtagCachedCountUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-cached-count-update {--limit=100}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update cached counter of hashtags';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $limit = $this->option('limit');
+        $tags = Hashtag::whereNull('cached_count')->limit($limit)->get();
+        $count = count($tags);
+        if(!$count) {
+            return;
+        }
+
+        $bar = $this->output->createProgressBar($count);
+        $bar->start();
+
+        foreach($tags as $tag) {
+            $count = DB::table('status_hashtags')->whereHashtagId($tag->id)->count();
+            if(!$count) {
+                $tag->cached_count = 0;
+                $tag->saveQuietly();
+                $bar->advance();
+                continue;
+            }
+            $tag->cached_count = $count;
+            $tag->saveQuietly();
+            $bar->advance();
+        }
+        $bar->finish();
+        $this->line(' ');
+        return;
+    }
+}

app/Console/Commands/HashtagRelatedGenerate.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+use App\Services\HashtagRelatedService;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use function Laravel\Prompts\multiselect;
+use function Laravel\Prompts\confirm;
+
+class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-related-generate {tag}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'tag' => 'Which hashtag should we generate related tags for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $tag = $this->argument('tag');
+        $hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
+        if(!$hashtag) {
+            $this->error('Hashtag not found, aborting...');
+            exit;
+        }
+
+        $exists = HashtagRelated::whereHashtagId($hashtag->id)->exists();
+
+        if($exists) {
+            $confirmed = confirm('Found existing related tags, do you want to regenerate them?');
+            if(!$confirmed) {
+                $this->error('Aborting...');
+                exit;
+            }
+        }
+
+        $this->info('Looking up #' . $tag . '...');
+
+        $tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
+        if(!$tags || $tags < 100) {
+            $this->error('Not enough posts found to generate related hashtags!');
+            exit;
+        }
+
+        $this->info('Found ' . $tags . ' posts that use that hashtag');
+        $related = collect(HashtagRelatedService::fetchRelatedTags($tag));
+
+        $selected = multiselect(
+            label: 'Which tags do you want to generate?',
+            options: $related->pluck('name'),
+            required: true,
+        );
+
+        $filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
+        $agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
+
+        HashtagRelated::updateOrCreate([
+            'hashtag_id' => $hashtag->id,
+        ], [
+            'related_tags' => array_values($filtered),
+            'agg_score' => $agg_score,
+            'last_calculated_at' => now()
+        ]);
+
+        $this->info('Finished!');
+    }
+}

app/Console/Commands/MediaCloudUrlRewrite.php

@@ -0,0 +1,140 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Media;
+use Cache, Storage;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+
+class MediaCloudUrlRewrite extends Command implements PromptsForMissingInput
+{
+    /**
+    * The name and signature of the console command.
+    *
+    * @var string
+    */
+    protected $signature = 'media:cloud-url-rewrite {oldDomain} {newDomain}';
+
+    /**
+    * Prompt for missing input arguments using the returned questions.
+    *
+    * @return array
+    */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'oldDomain' => 'The old S3 domain',
+            'newDomain' => 'The new S3 domain'
+        ];
+    }
+    /**
+    * The console command description.
+    *
+    * @var string
+    */
+    protected $description = 'Rewrite S3 media urls from local users';
+
+    /**
+    * Execute the console command.
+    */
+    public function handle()
+    {
+        $this->preflightCheck();
+        $this->bootMessage();
+        $this->confirmCloudUrl();
+    }
+
+    protected function preflightCheck()
+    {
+        if(config_cache('pixelfed.cloud_storage') != true) {
+            $this->info('Error: Cloud storage is not enabled!');
+            $this->error('Aborting...');
+            exit;
+        }
+    }
+
+    protected function bootMessage()
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Media Cloud Url Rewrite Tool');
+        $this->info('    ===');
+        $this->info('    Old S3: ' . trim($this->argument('oldDomain')));
+        $this->info('    New S3: ' . trim($this->argument('newDomain')));
+        $this->info(' ');
+    }
+
+    protected function confirmCloudUrl()
+    {
+        $disk = Storage::disk(config('filesystems.cloud'))->url('test');
+        $domain = parse_url($disk, PHP_URL_HOST);
+        if(trim($this->argument('newDomain')) !== $domain) {
+            $this->error('Error: The new S3 domain you entered is not currently configured');
+            exit;
+        }
+
+        if(!$this->confirm('Confirm this is correct')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        $this->updateUrls();
+    }
+
+    protected function updateUrls()
+    {
+        $this->info('Updating urls...');
+        $oldDomain = trim($this->argument('oldDomain'));
+        $newDomain = trim($this->argument('newDomain'));
+        $disk = Storage::disk(config('filesystems.cloud'));
+        $count = Media::whereNotNull('cdn_url')->count();
+        $bar = $this->output->createProgressBar($count);
+        $counter = 0;
+        $bar->start();
+        foreach(Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) {
+            if(strncmp($media->media_path, 'http', 4) === 0) {
+                $bar->advance();
+                continue;
+            }
+            $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST);
+            if($oldDomain != $cdnHost || $newDomain == $cdnHost) {
+                $bar->advance();
+                continue;
+            }
+
+            $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url);
+
+            if($media->thumbnail_url != null) {
+                $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST);
+                if($thumbHost == $oldDomain) {
+                    $thumbUrl = $disk->url($media->thumbnail_path);
+                    $media->thumbnail_url = $thumbUrl;
+                }
+            }
+
+            if($media->optimized_url != null) {
+                $optiHost = parse_url($media->optimized_url, PHP_URL_HOST);
+                if($optiHost == $oldDomain) {
+                    $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url);
+                    $media->optimized_url = $optiUrl;
+                }
+            }
+
+            $media->save();
+            $counter++;
+            $bar->advance();
+        }
+
+        $bar->finish();
+
+        $this->line(' ');
+        $this->info('Finished! Updated ' . $counter . ' total records!');
+        $this->line(' ');
+        $this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
+    }
+}

app/Console/Commands/NotificationEpochUpdate.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
+
+class NotificationEpochUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:notification-epoch-update';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update notification epoch';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        NotificationEpochUpdatePipeline::dispatch();
+    }
+}

app/Console/Commands/UserAdmin.php

@@ -3,16 +3,17 @@
 namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
 use App\User;
 
-class UserAdmin extends Command
+class UserAdmin extends Command implements PromptsForMissingInput
 {
     /**
      * The name and signature of the console command.
      *
      * @var string
      */
-    protected $signature = 'user:admin {id}';
+    protected $signature = 'user:admin {username}';
 
     /**
      * The console command description.
@@ -22,13 +23,15 @@ class UserAdmin extends Command
     protected $description = 'Make a user an admin, or remove admin privileges.';
 
     /**
-     * Create a new command instance.
+     * Prompt for missing input arguments using the returned questions.
      *
-     * @return void
+     * @return array
      */
-    public function __construct()
+    protected function promptForMissingArgumentsUsing()
     {
-        parent::__construct();
+        return [
+            'username' => 'Which username should we toggle admin privileges for?',
+        ];
     }
 
     /**
@@ -38,16 +41,15 @@ class UserAdmin extends Command
      */
     public function handle()
     {
-        $id = $this->argument('id');
-        if(ctype_digit($id) == true) {
-            $user = User::find($id);
-        } else {
-            $user = User::whereUsername($id)->first();
-        }
+        $id = $this->argument('username');
+
+        $user = User::whereUsername($id)->first();
+
         if(!$user) {
             $this->error('Could not find any user with that username or id.');
             exit;
         }
+
         $this->info('Found username: ' . $user->username);
         $state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?';
         $confirmed = $this->confirm($state);

app/Console/Commands/UserToggle2FA.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use App\User;
+
+class UserToggle2FA extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'user:2fa {username}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Disable two factor authentication for given username';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'username' => 'Which username should we disable 2FA for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $user = User::whereUsername($this->argument('username'))->first();
+
+        if(!$user) {
+            $this->error('Could not find any user with that username');
+            exit;
+        }
+
+        if(!$user->{'2fa_enabled'}) {
+            $this->info('User did not have 2FA enabled!');
+            return;
+        }
+
+        $user->{'2fa_enabled'} = false;
+        $user->{'2fa_secret'} = null;
+        $user->{'2fa_backup_codes'} = null;
+        $user->save();
+
+        $this->info('Successfully disabled 2FA on this account!');
+    }
+}

app/Console/Kernel.php

@@ -43,6 +43,9 @@ class Kernel extends ConsoleKernel
 			$schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37)->onOneServer();
 			$schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32)->onOneServer();
 		}
+
+		$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
+		$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);
 	}
 
 	/**

app/Http/Controllers/Admin/AdminReportController.php

@@ -643,7 +643,7 @@ trait AdminReportController
 				$q->whereNull('admin_seen') :
 				$q->whereNotNull('admin_seen');
 			})
-			->groupBy(['object_id', 'object_type'])
+			->groupBy(['id', 'object_id', 'object_type', 'profile_id'])
 			->cursorPaginate(6)
 			->withQueryString()
 		);

app/Http/Controllers/AdminShadowFilterController.php

@@ -0,0 +1,123 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+use App\Models\AdminShadowFilter;
+use App\Profile;
+use App\Services\AccountService;
+use App\Services\AdminShadowFilterService;
+
+class AdminShadowFilterController extends Controller
+{
+    public function __construct()
+    {
+        $this->middleware(['auth','admin']);
+    }
+
+    public function home(Request $request)
+    {
+        $filter = $request->input('filter');
+        $searchQuery = $request->input('q');
+        $filters = AdminShadowFilter::whereHas('profile')
+        ->when($filter, function($q, $filter) {
+            if($filter == 'all') {
+                return $q;
+            } else if($filter == 'inactive') {
+                return $q->whereActive(false);
+            } else {
+                return $q;
+            }
+        }, function($q, $filter) {
+            return $q->whereActive(true);
+        })
+        ->when($searchQuery, function($q, $searchQuery) {
+            $ids = Profile::where('username', 'like', '%' . $searchQuery . '%')
+                ->limit(100)
+                ->pluck('id')
+                ->toArray();
+            return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids);
+        })
+        ->latest()
+        ->paginate(10)
+        ->withQueryString();
+
+        return view('admin.asf.home', compact('filters'));
+    }
+
+    public function create(Request $request)
+    {
+        return view('admin.asf.create');
+    }
+
+    public function edit(Request $request, $id)
+    {
+        $filter = AdminShadowFilter::findOrFail($id);
+        $profile = AccountService::get($filter->item_id);
+        return view('admin.asf.edit', compact('filter', 'profile'));
+    }
+
+    public function store(Request $request)
+    {
+        $this->validate($request, [
+            'username' => 'required',
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $profile = Profile::whereUsername($request->input('username'))->first();
+
+        if(!$profile) {
+            return back()->withErrors(['Invalid account']);
+        }
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active') && $request->has('hide_from_public_feeds');
+
+        AdminShadowFilter::updateOrCreate([
+            'item_id' => $profile->id,
+            'item_type' => get_class($profile)
+        ], [
+            'is_local' => $profile->domain === null,
+            'note' => $request->input('note'),
+            'hide_from_public_feeds' => $request->has('hide_from_public_feeds'),
+            'admin_id' => $request->user()->profile_id,
+            'active' => $active
+        ]);
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+
+    public function storeEdit(Request $request, $id)
+    {
+        $this->validate($request, [
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $filter = AdminShadowFilter::findOrFail($id);
+
+        $profile = Profile::findOrFail($filter->item_id);
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active');
+        $filter->active = $active;
+        $filter->hide_from_public_feeds = $request->has('hide_from_public_feeds');
+        $filter->note = $request->input('note');
+        $filter->save();
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+}

app/Http/Controllers/Api/ApiV1Dot1Controller.php

@@ -11,6 +11,7 @@ use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 use App\AccountLog;
 use App\EmailVerification;
+use App\Follower;
 use App\Place;
 use App\Status;
 use App\Report;
@@ -19,8 +20,11 @@ use App\StatusArchived;
 use App\User;
 use App\UserSetting;
 use App\Services\AccountService;
+use App\Services\FollowerService;
 use App\Services\StatusService;
 use App\Services\ProfileStatusService;
+use App\Services\LikeService;
+use App\Services\ReblogService;
 use App\Services\PublicTimelineService;
 use App\Services\NetworkTimelineService;
 use App\Util\Lexer\RestrictedNames;
@@ -470,7 +474,7 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), 3, function(){}, 1800);
+		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), config('pixelfed.app_registration_rate_limit_attempts', 3), function(){}, config('pixelfed.app_registration_rate_limit_decay', 1800));
 		abort_if(!$rl, 400, 'Too many requests');
 
 		$this->validate($request, [
@@ -543,10 +547,10 @@ class ApiV1Dot1Controller extends Controller
 		$user->password = Hash::make($password);
 		$user->register_source = 'app';
 		$user->app_register_ip = $request->ip();
-		$user->app_register_token = Str::random(32);
+		$user->app_register_token = Str::random(40);
 		$user->save();
 
-		$rtoken = Str::random(mt_rand(64, 70));
+		$rtoken = Str::random(64);
 
 		$verify = new EmailVerification();
 		$verify->user_id = $user->id;
@@ -555,7 +559,12 @@ class ApiV1Dot1Controller extends Controller
 		$verify->random_token = $rtoken;
 		$verify->save();
 
-		$appUrl = url('/api/v1.1/auth/iarer?ut=' . $user->app_register_token . '&rt=' . $rtoken);
+		$params = http_build_query([
+			'ut' => $user->app_register_token,
+			'rt' => $rtoken,
+			'ea' => base64_encode($user->email)
+		]);
+		$appUrl = url('/api/v1.1/auth/iarer?'. $params);
 
 		Mail::to($user->email)->send(new ConfirmAppEmail($verify, $appUrl));
 
@@ -568,14 +577,19 @@ class ApiV1Dot1Controller extends Controller
 	{
 		$this->validate($request, [
 			'ut' => 'required',
-			'rt' => 'required'
+			'rt' => 'required',
+			'ea' => 'required'
 		]);
-		if(config('pixelfed.bouncer.cloud_ips.ban_signups')) {
-			abort_if(BouncerService::checkIp($request->ip()), 404);
-		}
 		$ut = $request->input('ut');
 		$rt = $request->input('rt');
-		$url = 'pixelfed://confirm-account/'. $ut . '?rt=' . $rt;
+		$ea = $request->input('ea');
+		$params = http_build_query([
+			'ut' => $ut,
+			'rt' => $rt,
+			'domain' => config('pixelfed.domain.app'),
+			'ea' => $ea
+		]);
+		$url = 'pixelfed://confirm-account/'. $ut . '?' . $params;
 		return redirect()->away($url);
 	}
 
@@ -589,8 +603,8 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), 10, function(){}, 1800);
-		abort_if(!$rl, 400, 'Too many requests');
+		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), config('pixelfed.app_registration_confirm_rate_limit_attempts', 20), function(){}, config('pixelfed.app_registration_confirm_rate_limit_decay', 1800));
+		abort_if(!$rl, 429, 'Too many requests');
 
 		$this->validate($request, [
 			'user_token' => 'required',
@@ -884,4 +898,19 @@ class ApiV1Dot1Controller extends Controller
 
         return [200];
     }
+
+    public function getMutualAccounts(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+        $account = AccountService::get($id, true);
+        if(!$account || !isset($account['id'])) { return []; }
+        $res = collect(FollowerService::mutualAccounts($request->user()->profile_id, $id))
+            ->map(function($accountId) {
+                return AccountService::get($accountId, true);
+            })
+            ->filter()
+            ->take(24)
+            ->values();
+        return $this->json($res);
+    }
 }

app/Http/Controllers/Api/ApiV2Controller.php

@@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{
 use App\Transformer\Api\{
 	RelationshipTransformer,
 };
+use App\Util\Site\Nodeinfo;
 
 class ApiV2Controller extends Controller
 {
@@ -77,12 +78,7 @@ class ApiV2Controller extends Controller
 			'description' => config_cache('app.short_description'),
 			'usage' => [
 				'users' => [
-					'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() {
-						return User::select('last_active_at', 'created_at')
-							->where('last_active_at', '>', now()->subMonths(1))
-							->orWhere('created_at', '>', now()->subMonths(1))
-							->count();
-					})
+					'active_month' => (int) Nodeinfo::activeUsersMonthly()
 				]
 			],
 			'thumbnail' => [

app/Http/Controllers/Api/V1/DomainBlockController.php

@@ -0,0 +1,118 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Models\UserDomainBlock;
+use App\Util\ActivityPub\Helpers;
+use App\Services\UserFilterService;
+use Illuminate\Bus\Batch;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\Cache;
+use App\Jobs\HomeFeedPipeline\FeedRemoveDomainPipeline;
+use App\Jobs\ProfilePipeline\ProfilePurgeNotificationsByDomain;
+use App\Jobs\ProfilePipeline\ProfilePurgeFollowersByDomain;
+
+class DomainBlockController extends Controller
+{
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    public function index(Request $request)
+    {
+        abort_unless($request->user(), 403);
+        $this->validate($request, [
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+        $id = $request->user()->profile_id;
+        $filters = UserDomainBlock::whereProfileId($id)->orderByDesc('id')->cursorPaginate($limit);
+        $links = null;
+        $headers = [];
+
+        if($filters->nextCursor()) {
+            $links .= '<'.$filters->nextPageUrl().'&limit='.$limit.'>; rel="next"';
+        }
+
+        if($filters->previousCursor()) {
+            if($links != null) {
+                $links .= ', ';
+            }
+            $links .= '<'.$filters->previousPageUrl().'&limit='.$limit.'>; rel="prev"';
+        }
+
+        if($links) {
+            $headers = ['Link' => $links];
+        }
+        return $this->json($filters->pluck('domain'), 200, $headers);
+    }
+
+    public function store(Request $request)
+    {
+        abort_unless($request->user(), 403);
+
+        $this->validate($request, [
+            'domain' => 'required|active_url|min:1|max:120'
+        ]);
+
+        $pid = $request->user()->profile_id;
+
+        $domain = trim($request->input('domain'));
+
+        if(Helpers::validateUrl($domain) == false) {
+            return abort(500, 'Invalid domain or already blocked by server admins');
+        }
+
+        $domain = strtolower(parse_url($domain, PHP_URL_HOST));
+
+        abort_if(config_cache('pixelfed.domain.app') == $domain, 400, 'Cannot ban your own server');
+
+        $existingCount = UserDomainBlock::whereProfileId($pid)->count();
+        $maxLimit = config('instance.user_filters.max_domain_blocks');
+        $errorMsg =  __('profile.block.domain.max', ['max' => $maxLimit]);
+
+        abort_if($existingCount >= $maxLimit, 400, $errorMsg);
+
+        $block = UserDomainBlock::updateOrCreate([
+            'profile_id' => $pid,
+            'domain' => $domain
+        ]);
+
+        if($block->wasRecentlyCreated) {
+            Bus::batch([
+                [
+                    new FeedRemoveDomainPipeline($pid, $domain),
+                    new ProfilePurgeNotificationsByDomain($pid, $domain),
+                    new ProfilePurgeFollowersByDomain($pid, $domain)
+                ]
+            ])->allowFailures()->onQueue('feed')->dispatch();
+
+            Cache::forget('profile:following:' . $pid);
+            UserFilterService::domainBlocks($pid, true);
+        }
+
+        return $this->json([]);
+    }
+
+    public function delete(Request $request)
+    {
+        abort_unless($request->user(), 403);
+
+        $this->validate($request, [
+            'domain' => 'required|min:1|max:120'
+        ]);
+
+        $pid = $request->user()->profile_id;
+
+        $domain = strtolower(trim($request->input('domain')));
+
+        $filters = UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->delete();
+
+        UserFilterService::domainBlocks($pid, true);
+
+        return $this->json([]);
+    }
+}

app/Http/Controllers/Api/V1/TagsController.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Hashtag;
+use App\HashtagFollow;
+use App\StatusHashtag;
+use App\Services\AccountService;
+use App\Services\HashtagService;
+use App\Services\HashtagFollowService;
+use App\Services\HashtagRelatedService;
+use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+
+class TagsController extends Controller
+{
+    const PF_API_ENTITY_KEY = "_pe";
+
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    /**
+    * GET /api/v1/tags/:id/related
+    *
+    *
+    * @return array
+    */
+    public function relatedTags(Request $request, $tag)
+    {
+        abort_unless($request->user(), 403);
+        $tag = Hashtag::whereSlug($tag)->firstOrFail();
+        return HashtagRelatedService::get($tag->id);
+    }
+
+    /**
+    * POST /api/v1/tags/:id/follow
+    *
+    *
+    * @return object
+    */
+    public function followHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        abort_if(
+            HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
+            422,
+            'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
+        );
+
+        $follows = HashtagFollow::updateOrCreate(
+            [
+                'profile_id' => $account['id'],
+                'hashtag_id' => $tag->id
+            ],
+            [
+                'user_id' => $request->user()->id
+            ]
+        );
+
+        HashtagService::follow($pid, $tag->id);
+        HashtagFollowService::add($tag->id, $pid);
+
+        return response()->json(FollowedTagResource::make($follows)->toArray($request));
+    }
+
+    /**
+    * POST /api/v1/tags/:id/unfollow
+    *
+    *
+    * @return object
+    */
+    public function unfollowHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        $follows = HashtagFollow::whereProfileId($pid)
+            ->whereHashtagId($tag->id)
+            ->first();
+
+        if(!$follows) {
+            return [
+                'name' => $tag->name,
+                'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        if($follows) {
+            HashtagService::unfollow($pid, $tag->id);
+            HashtagFollowService::unfollow($tag->id, $pid);
+            HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
+            $follows->delete();
+        }
+
+        $res = FollowedTagResource::make($follows)->toArray($request);
+        $res['following'] = false;
+        return response()->json($res);
+    }
+
+    /**
+    * GET /api/v1/tags/:id
+    *
+    *
+    * @return object
+    */
+    public function getHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        if(!$tag) {
+            return [
+                'name' => $id,
+                'url' => config('app.url') . '/i/web/hashtag/' . $id,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        $res = [
+            'name' => $tag->name,
+            'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+            'history' => [],
+            'following' => HashtagService::isFollowing($pid, $tag->id)
+        ];
+
+        if($request->has(self::PF_API_ENTITY_KEY)) {
+            $res['count'] = HashtagService::count($tag->id);
+        }
+
+        return $this->json($res);
+    }
+
+    /**
+    * GET /api/v1/followed_tags
+    *
+    *
+    * @return array
+    */
+    public function getFollowedTags(Request $request)
+    {
+        abort_if(!$request->user(), 403);
+
+        $account = AccountService::get($request->user()->profile_id);
+
+        $this->validate($request, [
+            'cursor' => 'sometimes',
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+
+        $res = HashtagFollow::whereProfileId($account['id'])
+            ->orderByDesc('id')
+            ->cursorPaginate($limit)
+            ->withQueryString();
+
+        $pagination = false;
+        $prevPage = $res->nextPageUrl();
+        $nextPage = $res->previousPageUrl();
+        if($nextPage && $prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
+        } else if($nextPage && !$prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next"';
+        } else if(!$nextPage && $prevPage) {
+            $pagination = '<' . $prevPage . '>; rel="prev"';
+        }
+
+        if($pagination) {
+            return response()->json(FollowedTagResource::collection($res)->collection)
+                ->header('Link', $pagination);
+        }
+        return response()->json(FollowedTagResource::collection($res)->collection);
+    }
+}

app/Http/Controllers/CollectionController.php

@@ -153,7 +153,7 @@ class CollectionController extends Controller
             abort(400, 'You can only add '.$max.' posts per collection');
         }
 
-        $status = Status::whereScope('public')
+        $status = Status::whereIn('scope', ['public', 'unlisted'])
             ->whereProfileId($profileId)
             ->whereIn('type', ['photo', 'photo:album', 'video'])
             ->findOrFail($postId);
@@ -166,17 +166,13 @@ class CollectionController extends Controller
             'order'         => $count,
         ]);
 
-        CollectionService::addItem(
-        	$collection->id,
-        	$status->id,
-        	$count
-        );
+        CollectionService::deleteCollection($collection->id);
 
         $collection->updated_at = now();
         $collection->save();
         CollectionService::setCollection($collection->id, $collection);
 
-        return StatusService::get($status->id);
+        return StatusService::get($status->id, false);
     }
 
     public function getCollection(Request $request, $id)
@@ -226,10 +222,10 @@ class CollectionController extends Controller
 
         return collect($items)
         	->map(function($id) {
-        		return StatusService::get($id);
+                return StatusService::get($id, false);
         	})
         	->filter(function($item) {
-        		return $item && isset($item['account'], $item['media_attachments']);
+                return $item && ($item['visibility'] == 'public' ||  $item['visibility'] == 'unlisted') && isset($item['account'], $item['media_attachments']);
         	})
         	->values();
     }
@@ -298,7 +294,7 @@ class CollectionController extends Controller
             abort(400, 'You cannot delete the only post of a collection!');
         }
 
-        $status = Status::whereScope('public')
+        $status = Status::whereIn('scope', ['public', 'unlisted'])
             ->whereIn('type', ['photo', 'photo:album', 'video'])
             ->findOrFail($postId);
 

app/Http/Controllers/ComposeController.php

@@ -415,7 +415,7 @@ class ComposeController extends Controller
 		$results = Profile::select('id','domain','username')
 			->whereNotIn('id', $blocked)
 			->where('username','like','%'.$q.'%')
-			->groupBy('domain')
+			->groupBy('id', 'domain')
 			->limit(15)
 			->get()
 			->map(function($profile) {

app/Http/Controllers/DirectMessageController.php

@@ -17,12 +17,15 @@ use App\{
 use App\Services\MediaPathService;
 use App\Services\MediaBlocklistService;
 use App\Jobs\StatusPipeline\NewStatusPipeline;
+use App\Jobs\StatusPipeline\StatusDelete;
 use Illuminate\Support\Str;
 use App\Util\ActivityPub\Helpers;
 use App\Services\AccountService;
 use App\Services\StatusService;
 use App\Services\WebfingerService;
 use App\Models\Conversation;
+use App\Jobs\DirectPipeline\DirectDeletePipeline;
+use App\Jobs\DirectPipeline\DirectDeliverPipeline;
 
 class DirectMessageController extends Controller
 {
@@ -500,6 +503,8 @@ class DirectMessageController extends Controller
 		if($recipient['local'] == false) {
 			$dmc = $dm;
 			$this->remoteDelete($dmc);
+		} else {
+			StatusDelete::dispatch($status)->onQueue('high');
 		}
 
 		if(Conversation::whereStatusId($sid)->count()) {
@@ -541,9 +546,7 @@ class DirectMessageController extends Controller
 
 		StatusService::del($status->id, true);
 
-		$status->delete();
-		$dm->delete();
-
+		$status->forceDeleteQuietly();
 		return [200];
 	}
 
@@ -829,7 +832,7 @@ class DirectMessageController extends Controller
 			]
 		];
 
-		Helpers::sendSignedObject($profile, $url, $body);
+		DirectDeliverPipeline::dispatch($profile, $url, $body)->onQueue('high');
 	}
 
 	public function remoteDelete($dm)
@@ -852,7 +855,6 @@ class DirectMessageController extends Controller
 				'type' => 'Tombstone'
 			]
 		];
-
-		Helpers::sendSignedObject($profile, $url, $body);
+		DirectDeletePipeline::dispatch($profile, $url, $body)->onQueue('high');
 	}
 }

app/Http/Controllers/FederationController.php

@@ -3,17 +3,17 @@
 namespace App\Http\Controllers;
 
 use App\Jobs\InboxPipeline\{
-	DeleteWorker,
-	InboxWorker,
-	InboxValidator
+    DeleteWorker,
+    InboxWorker,
+    InboxValidator
 };
 use App\Jobs\RemoteFollowPipeline\RemoteFollowPipeline;
 use App\{
-	AccountLog,
-	Like,
-	Profile,
-	Status,
-	User
+    AccountLog,
+    Like,
+    Profile,
+    Status,
+    User
 };
 use App\Util\Lexer\Nickname;
 use App\Util\Webfinger\Webfinger;
@@ -24,243 +24,251 @@ use Illuminate\Http\Request;
 use League\Fractal;
 use App\Util\Site\Nodeinfo;
 use App\Util\ActivityPub\{
-	Helpers,
-	HttpSignature,
-	Outbox
+    Helpers,
+    HttpSignature,
+    Outbox
 };
 use Zttp\Zttp;
 use App\Services\InstanceService;
+use App\Services\AccountService;
 
 class FederationController extends Controller
 {
-	public function nodeinfoWellKnown()
-	{
-		abort_if(!config('federation.nodeinfo.enabled'), 404);
-		return response()->json(Nodeinfo::wellKnown(), 200, [], JSON_UNESCAPED_SLASHES)
-			->header('Access-Control-Allow-Origin','*');
-	}
+    public function nodeinfoWellKnown()
+    {
+        abort_if(!config('federation.nodeinfo.enabled'), 404);
+        return response()->json(Nodeinfo::wellKnown(), 200, [], JSON_UNESCAPED_SLASHES)
+            ->header('Access-Control-Allow-Origin','*');
+    }
 
-	public function nodeinfo()
-	{
-		abort_if(!config('federation.nodeinfo.enabled'), 404);
-		return response()->json(Nodeinfo::get(), 200, [], JSON_UNESCAPED_SLASHES)
-			->header('Access-Control-Allow-Origin','*');
-	}
+    public function nodeinfo()
+    {
+        abort_if(!config('federation.nodeinfo.enabled'), 404);
+        return response()->json(Nodeinfo::get(), 200, [], JSON_UNESCAPED_SLASHES)
+            ->header('Access-Control-Allow-Origin','*');
+    }
 
-	public function webfinger(Request $request)
-	{
-		if (!config('federation.webfinger.enabled') ||
-			!$request->has('resource') ||
-			!$request->filled('resource')
-		) {
-			return response('', 400);
-		}
+    public function webfinger(Request $request)
+    {
+        if (!config('federation.webfinger.enabled') ||
+            !$request->has('resource') ||
+            !$request->filled('resource')
+        ) {
+            return response('', 400);
+        }
 
-		$resource = $request->input('resource');
-		$domain = config('pixelfed.domain.app');
+        $resource = $request->input('resource');
+        $domain = config('pixelfed.domain.app');
 
-		if(config('federation.activitypub.sharedInbox') &&
-			$resource == 'acct:' . $domain . '@' . $domain) {
-			$res = [
-				'subject' => 'acct:' . $domain . '@' . $domain,
-				'aliases' => [
-					'https://' . $domain . '/i/actor'
-				],
-				'links' => [
-					[
-						'rel' => 'http://webfinger.net/rel/profile-page',
-						'type' => 'text/html',
-						'href' => 'https://' . $domain . '/site/kb/instance-actor'
-					],
-					[
-						'rel' => 'self',
-						'type' => 'application/activity+json',
-						'href' => 'https://' . $domain . '/i/actor'
-					]
-				]
-			];
-			return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
-		}
-		$hash = hash('sha256', $resource);
-		$key = 'federation:webfinger:sha256:' . $hash;
-		if($cached = Cache::get($key)) {
-			return response()->json($cached, 200, [], JSON_UNESCAPED_SLASHES);
-		}
-		if(strpos($resource, $domain) == false) {
-			return response('', 400);
-		}
-		$parsed = Nickname::normalizeProfileUrl($resource);
-		if(empty($parsed) || $parsed['domain'] !== $domain) {
-			return response('', 400);
-		}
-		$username = $parsed['username'];
-		$profile = Profile::whereNull('domain')->whereUsername($username)->first();
-		if(!$profile || $profile->status !== null) {
-			return response('', 400);
-		}
-		$webfinger = (new Webfinger($profile))->generate();
-		Cache::put($key, $webfinger, 1209600);
+        if(config('federation.activitypub.sharedInbox') &&
+            $resource == 'acct:' . $domain . '@' . $domain) {
+            $res = [
+                'subject' => 'acct:' . $domain . '@' . $domain,
+                'aliases' => [
+                    'https://' . $domain . '/i/actor'
+                ],
+                'links' => [
+                    [
+                        'rel' => 'http://webfinger.net/rel/profile-page',
+                        'type' => 'text/html',
+                        'href' => 'https://' . $domain . '/site/kb/instance-actor'
+                    ],
+                    [
+                        'rel' => 'self',
+                        'type' => 'application/activity+json',
+                        'href' => 'https://' . $domain . '/i/actor'
+                    ]
+                ]
+            ];
+            return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
+        }
+        $hash = hash('sha256', $resource);
+        $key = 'federation:webfinger:sha256:' . $hash;
+        if($cached = Cache::get($key)) {
+            return response()->json($cached, 200, [], JSON_UNESCAPED_SLASHES);
+        }
+        if(strpos($resource, $domain) == false) {
+            return response('', 400);
+        }
+        $parsed = Nickname::normalizeProfileUrl($resource);
+        if(empty($parsed) || $parsed['domain'] !== $domain) {
+            return response('', 400);
+        }
+        $username = $parsed['username'];
+        $profile = Profile::whereNull('domain')->whereUsername($username)->first();
+        if(!$profile || $profile->status !== null) {
+            return response('', 400);
+        }
+        $webfinger = (new Webfinger($profile))->generate();
+        Cache::put($key, $webfinger, 1209600);
 
-		return response()->json($webfinger, 200, [], JSON_UNESCAPED_SLASHES)
-			->header('Access-Control-Allow-Origin','*');
-	}
+        return response()->json($webfinger, 200, [], JSON_UNESCAPED_SLASHES)
+            ->header('Access-Control-Allow-Origin','*');
+    }
 
-	public function hostMeta(Request $request)
-	{
-		abort_if(!config('federation.webfinger.enabled'), 404);
+    public function hostMeta(Request $request)
+    {
+        abort_if(!config('federation.webfinger.enabled'), 404);
 
-		$path = route('well-known.webfinger');
-		$xml = '<?xml version="1.0" encoding="UTF-8"?><XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="'.$path.'?resource={uri}"/></XRD>';
+        $path = route('well-known.webfinger');
+        $xml = '<?xml version="1.0" encoding="UTF-8"?><XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="'.$path.'?resource={uri}"/></XRD>';
 
-		return response($xml)->header('Content-Type', 'application/xrd+xml');
-	}
+        return response($xml)->header('Content-Type', 'application/xrd+xml');
+    }
 
-	public function userOutbox(Request $request, $username)
-	{
-		abort_if(!config_cache('federation.activitypub.enabled'), 404);
+    public function userOutbox(Request $request, $username)
+    {
+        abort_if(!config_cache('federation.activitypub.enabled'), 404);
 
-		if(!$request->wantsJson()) {
-			return redirect('/' . $username);
-		}
+        if(!$request->wantsJson()) {
+            return redirect('/' . $username);
+        }
 
-		$res = [
-			'@context' => 'https://www.w3.org/ns/activitystreams',
-			'id' => 'https://' . config('pixelfed.domain.app') . '/users/' . $username . '/outbox',
-			'type' => 'OrderedCollection',
-			'totalItems' => 0,
-			'orderedItems' => []
-		];
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['statuses_count']), 404);
+        $res = [
+            '@context' => 'https://www.w3.org/ns/activitystreams',
+            'id' => 'https://' . config('pixelfed.domain.app') . '/users/' . $username . '/outbox',
+            'type' => 'OrderedCollection',
+            'totalItems' => $account['statuses_count'] ?? 0,
+        ];
 
-		return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json');
-	}
+        return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json');
+    }
 
-	public function userInbox(Request $request, $username)
-	{
-		abort_if(!config_cache('federation.activitypub.enabled'), 404);
-		abort_if(!config('federation.activitypub.inbox'), 404);
+    public function userInbox(Request $request, $username)
+    {
+        abort_if(!config_cache('federation.activitypub.enabled'), 404);
+        abort_if(!config('federation.activitypub.inbox'), 404);
 
-		$headers = $request->headers->all();
-		$payload = $request->getContent();
-		if(!$payload || empty($payload)) {
-			return;
-		}
-		$obj = json_decode($payload, true, 8);
-		if(!isset($obj['id'])) {
-			return;
-		}
-		$domain = parse_url($obj['id'], PHP_URL_HOST);
-		if(in_array($domain, InstanceService::getBannedDomains())) {
-			return;
-		}
+        $headers = $request->headers->all();
+        $payload = $request->getContent();
+        if(!$payload || empty($payload)) {
+            return;
+        }
+        $obj = json_decode($payload, true, 8);
+        if(!isset($obj['id'])) {
+            return;
+        }
+        $domain = parse_url($obj['id'], PHP_URL_HOST);
+        if(in_array($domain, InstanceService::getBannedDomains())) {
+            return;
+        }
 
-		if(isset($obj['type']) && $obj['type'] === 'Delete') {
-			if(isset($obj['object']) && isset($obj['object']['type']) && isset($obj['object']['id'])) {
-				if($obj['object']['type'] === 'Person') {
-					if(Profile::whereRemoteUrl($obj['object']['id'])->exists()) {
-						dispatch(new DeleteWorker($headers, $payload))->onQueue('inbox');
-						return;
-					}
-				}
+        if(isset($obj['type']) && $obj['type'] === 'Delete') {
+            if(isset($obj['object']) && isset($obj['object']['type']) && isset($obj['object']['id'])) {
+                if($obj['object']['type'] === 'Person') {
+                    if(Profile::whereRemoteUrl($obj['object']['id'])->exists()) {
+                        dispatch(new DeleteWorker($headers, $payload))->onQueue('inbox');
+                        return;
+                    }
+                }
 
-				if($obj['object']['type'] === 'Tombstone') {
-					if(Status::whereObjectUrl($obj['object']['id'])->exists()) {
-						dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
-						return;
-					}
-				}
+                if($obj['object']['type'] === 'Tombstone') {
+                    if(Status::whereObjectUrl($obj['object']['id'])->exists()) {
+                        dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
+                        return;
+                    }
+                }
 
-				if($obj['object']['type'] === 'Story') {
-					dispatch(new DeleteWorker($headers, $payload))->onQueue('story');
-					return;
-				}
-			}
-			return;
-		} else if( isset($obj['type']) && in_array($obj['type'], ['Follow', 'Accept'])) {
-			dispatch(new InboxValidator($username, $headers, $payload))->onQueue('follow');
-		} else {
-			dispatch(new InboxValidator($username, $headers, $payload))->onQueue('high');
-		}
-		return;
-	}
+                if($obj['object']['type'] === 'Story') {
+                    dispatch(new DeleteWorker($headers, $payload))->onQueue('story');
+                    return;
+                }
+            }
+            return;
+        } else if( isset($obj['type']) && in_array($obj['type'], ['Follow', 'Accept'])) {
+            dispatch(new InboxValidator($username, $headers, $payload))->onQueue('follow');
+        } else {
+            dispatch(new InboxValidator($username, $headers, $payload))->onQueue('high');
+        }
+        return;
+    }
 
-	public function sharedInbox(Request $request)
-	{
-		abort_if(!config_cache('federation.activitypub.enabled'), 404);
-		abort_if(!config('federation.activitypub.sharedInbox'), 404);
+    public function sharedInbox(Request $request)
+    {
+        abort_if(!config_cache('federation.activitypub.enabled'), 404);
+        abort_if(!config('federation.activitypub.sharedInbox'), 404);
 
-		$headers = $request->headers->all();
-		$payload = $request->getContent();
+        $headers = $request->headers->all();
+        $payload = $request->getContent();
 
-		if(!$payload || empty($payload)) {
-			return;
-		}
+        if(!$payload || empty($payload)) {
+            return;
+        }
 
-		$obj = json_decode($payload, true, 8);
-		if(!isset($obj['id'])) {
-			return;
-		}
+        $obj = json_decode($payload, true, 8);
+        if(!isset($obj['id'])) {
+            return;
+        }
 
-		$domain = parse_url($obj['id'], PHP_URL_HOST);
-		if(in_array($domain, InstanceService::getBannedDomains())) {
-			return;
-		}
+        $domain = parse_url($obj['id'], PHP_URL_HOST);
+        if(in_array($domain, InstanceService::getBannedDomains())) {
+            return;
+        }
 
-		if(isset($obj['type']) && $obj['type'] === 'Delete') {
-			if(isset($obj['object']) && isset($obj['object']['type']) && isset($obj['object']['id'])) {
-				if($obj['object']['type'] === 'Person') {
-					if(Profile::whereRemoteUrl($obj['object']['id'])->exists()) {
-						dispatch(new DeleteWorker($headers, $payload))->onQueue('inbox');
-						return;
-					}
-				}
+        if(isset($obj['type']) && $obj['type'] === 'Delete') {
+            if(isset($obj['object']) && isset($obj['object']['type']) && isset($obj['object']['id'])) {
+                if($obj['object']['type'] === 'Person') {
+                    if(Profile::whereRemoteUrl($obj['object']['id'])->exists()) {
+                        dispatch(new DeleteWorker($headers, $payload))->onQueue('inbox');
+                        return;
+                    }
+                }
 
-				if($obj['object']['type'] === 'Tombstone') {
-					if(Status::whereObjectUrl($obj['object']['id'])->exists()) {
-						dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
-						return;
-					}
-				}
+                if($obj['object']['type'] === 'Tombstone') {
+                    if(Status::whereObjectUrl($obj['object']['id'])->exists()) {
+                        dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
+                        return;
+                    }
+                }
 
-				if($obj['object']['type'] === 'Story') {
-					dispatch(new DeleteWorker($headers, $payload))->onQueue('story');
-					return;
-				}
-			}
-			return;
-		} else if( isset($obj['type']) && in_array($obj['type'], ['Follow', 'Accept'])) {
-			dispatch(new InboxWorker($headers, $payload))->onQueue('follow');
-		} else {
-			dispatch(new InboxWorker($headers, $payload))->onQueue('shared');
-		}
-		return;
-	}
+                if($obj['object']['type'] === 'Story') {
+                    dispatch(new DeleteWorker($headers, $payload))->onQueue('story');
+                    return;
+                }
+            }
+            return;
+        } else if( isset($obj['type']) && in_array($obj['type'], ['Follow', 'Accept'])) {
+            dispatch(new InboxWorker($headers, $payload))->onQueue('follow');
+        } else {
+            dispatch(new InboxWorker($headers, $payload))->onQueue('shared');
+        }
+        return;
+    }
 
-	public function userFollowing(Request $request, $username)
-	{
-		abort_if(!config_cache('federation.activitypub.enabled'), 404);
+    public function userFollowing(Request $request, $username)
+    {
+        abort_if(!config_cache('federation.activitypub.enabled'), 404);
 
-		$obj = [
-			'@context' => 'https://www.w3.org/ns/activitystreams',
-			'id'       => $request->getUri(),
-			'type'     => 'OrderedCollectionPage',
-			'totalItems' => 0,
-			'orderedItems' => []
-		];
-		return response()->json($obj);
-	}
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['following_count']), 404);
+        $obj = [
+            '@context' => 'https://www.w3.org/ns/activitystreams',
+            'id'       => $request->getUri(),
+            'type'     => 'OrderedCollection',
+            'totalItems' => $account['following_count'] ?? 0,
+        ];
+        return response()->json($obj);
+    }
 
-	public function userFollowers(Request $request, $username)
-	{
-		abort_if(!config_cache('federation.activitypub.enabled'), 404);
-
-		$obj = [
-			'@context' => 'https://www.w3.org/ns/activitystreams',
-			'id'       => $request->getUri(),
-			'type'     => 'OrderedCollectionPage',
-			'totalItems' => 0,
-			'orderedItems' => []
-		];
-
-		return response()->json($obj);
-	}
+    public function userFollowers(Request $request, $username)
+    {
+        abort_if(!config_cache('federation.activitypub.enabled'), 404);
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['followers_count']), 404);
+        $obj = [
+            '@context' => 'https://www.w3.org/ns/activitystreams',
+            'id'       => $request->getUri(),
+            'type'     => 'OrderedCollection',
+            'totalItems' => $account['followers_count'] ?? 0,
+        ];
+        return response()->json($obj);
+    }
 }

app/Http/Controllers/ImportPostController.php

@@ -83,6 +83,17 @@ class ImportPostController extends Controller
         );
     }
 
+    public function formatHashtags($val = false)
+    {
+        if(!$val || !strlen($val)) {
+            return null;
+        }
+
+        $groupedHashtagRegex = '/#\w+(?=#)/';
+
+        return preg_replace($groupedHashtagRegex, '$0 ', $val);
+    }
+
     public function store(Request $request)
     {
         abort_unless(config('import.instagram.enabled'), 404);
@@ -128,11 +139,11 @@ class ImportPostController extends Controller
             $ip->media = $c->map(function($m) {
                 return [
                     'uri' => $m['uri'],
-                    'title' => $m['title'],
+                    'title' => $this->formatHashtags($m['title']),
                     'creation_timestamp' => $m['creation_timestamp']
                 ];
             })->toArray();
-            $ip->caption = $c->count() > 1 ? $file['title'] : $ip->media[0]['title'];
+            $ip->caption = $c->count() > 1 ? $this->formatHashtags($file['title']) : $this->formatHashtags($ip->media[0]['title']);
             $ip->filename = last(explode('/', $ip->media[0]['uri']));
             $ip->metadata = $c->map(function($m) {
                 return [

app/Http/Controllers/LikeController.php

@@ -25,8 +25,7 @@ class LikeController extends Controller
 			'item'    => 'required|integer|min:1',
 		]);
 
-		// API deprecated
-		return;
+		abort(422, 'Deprecated API Endpoint');
 
 		$user = Auth::user();
 		$profile = $user->profile;
@@ -34,7 +33,7 @@ class LikeController extends Controller
 
 		if (Like::whereStatusId($status->id)->whereProfileId($profile->id)->exists()) {
 			$like = Like::whereProfileId($profile->id)->whereStatusId($status->id)->firstOrFail();
-			UnlikePipeline::dispatch($like);
+			UnlikePipeline::dispatch($like)->onQueue('feed');
 		} else {
 			abort_if(
 				Like::whereProfileId($user->profile_id)
@@ -60,7 +59,7 @@ class LikeController extends Controller
 					]) == false;
 				$like->save();
 				$status->save();
-				LikePipeline::dispatch($like);
+				LikePipeline::dispatch($like)->onQueue('feed');
 			}
 		}
 

app/Http/Controllers/RemoteAuthController.php

@@ -23,7 +23,13 @@ class RemoteAuthController extends Controller
 {
     public function start(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -37,7 +43,13 @@ class RemoteAuthController extends Controller
 
     public function getAuthDomains(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
 
         if(config('remote-auth.mastodon.domains.only_custom')) {
             $res = config('remote-auth.mastodon.domains.custom');
@@ -69,7 +81,14 @@ class RemoteAuthController extends Controller
 
     public function redirect(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $this->validate($request, ['domain' => 'required']);
 
         $domain = $request->input('domain');
@@ -158,6 +177,14 @@ class RemoteAuthController extends Controller
 
     public function preflight(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) {
             return redirect('/login');
         }
@@ -167,6 +194,14 @@ class RemoteAuthController extends Controller
 
     public function handleCallback(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $domain = $request->session()->get('oauth_domain');
 
         if($request->filled('code')) {
@@ -195,7 +230,13 @@ class RemoteAuthController extends Controller
 
     public function onboarding(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -204,6 +245,13 @@ class RemoteAuthController extends Controller
 
     public function sessionCheck(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -248,6 +296,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonData(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -279,6 +334,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateUsername(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -334,6 +396,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateEmail(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -359,6 +428,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonFollowers(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -386,6 +462,13 @@ class RemoteAuthController extends Controller
 
     public function handleSubmit(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -464,7 +547,13 @@ class RemoteAuthController extends Controller
 
     public function storeBio(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -483,7 +572,13 @@ class RemoteAuthController extends Controller
 
     public function accountToId(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -525,7 +620,13 @@ class RemoteAuthController extends Controller
 
     public function storeAvatar(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         $this->validate($request, [
             'avatar_url' => 'required|active_url',
@@ -547,7 +648,13 @@ class RemoteAuthController extends Controller
 
     public function finishUp(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
 
         $currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app');
@@ -564,7 +671,13 @@ class RemoteAuthController extends Controller
 
     public function handleLogin(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);

app/Http/Controllers/Settings/PrivacySettings.php

@@ -14,19 +14,20 @@ use App\Util\Lexer\PrettyNumber;
 use App\Util\ActivityPub\Helpers;
 use Auth, Cache, DB;
 use Illuminate\Http\Request;
+use App\Models\UserDomainBlock;
 
 trait PrivacySettings
 {
 
     public function privacy()
     {
-		$user = Auth::user();
-		$settings = $user->settings;
-		$profile = $user->profile;
-		$is_private = $profile->is_private;
-		$settings['is_private'] = (bool) $is_private;
+        $user = Auth::user();
+        $settings = $user->settings;
+        $profile = $user->profile;
+        $is_private = $profile->is_private;
+        $settings['is_private'] = (bool) $is_private;
 
-		return view('settings.privacy', compact('settings', 'profile'));
+        return view('settings.privacy', compact('settings', 'profile'));
     }
 
     public function privacyStore(Request $request)
@@ -39,11 +40,13 @@ trait PrivacySettings
           'public_dm',
           'show_profile_follower_count',
           'show_profile_following_count',
+          'indexable',
           'show_atom',
         ];
 
-		$profile->is_suggestable = $request->input('is_suggestable') == 'on';
-		$profile->save();
+        $profile->indexable = $request->input('indexable') == 'on';
+        $profile->is_suggestable = $request->input('is_suggestable') == 'on';
+        $profile->save();
 
         foreach ($fields as $field) {
             $form = $request->input($field);
@@ -70,6 +73,8 @@ trait PrivacySettings
                 } else {
                     $settings->{$field} = false;
                 }
+            } elseif ($field == 'indexable') {
+
             } else {
                 if ($form == 'on') {
                     $settings->{$field} = true;
@@ -145,47 +150,25 @@ trait PrivacySettings
 
     public function blockedInstances()
     {
-        $pid = Auth::user()->profile->id;
-        $filters = UserFilter::whereUserId($pid)
-            ->whereFilterableType('App\Instance')
-            ->whereFilterType('block')
-            ->orderByDesc('id')
-            ->paginate(10);
-        return view('settings.privacy.blocked-instances', compact('filters'));
+        // deprecated
+        abort(404);
+    }
+
+    public function domainBlocks()
+    {
+        return view('settings.privacy.domain-blocks');
     }
 
     public function blockedInstanceStore(Request $request)
     {
-        $this->validate($request, [
-            'domain' => 'required|url|min:1|max:120'
-        ]);
-        $domain = $request->input('domain');
-        if(Helpers::validateUrl($domain) == false) {
-            return abort(400, 'Invalid domain');
-        }
-        $domain = parse_url($domain, PHP_URL_HOST);
-        $instance = Instance::firstOrCreate(['domain' => $domain]);
-        $filter = new UserFilter;
-        $filter->user_id = Auth::user()->profile->id;
-        $filter->filterable_id = $instance->id;
-        $filter->filterable_type = 'App\Instance';
-        $filter->filter_type = 'block';
-        $filter->save();
-        return response()->json(['msg' => 200]);
+        // deprecated
+        abort(404);
     }
 
     public function blockedInstanceUnblock(Request $request)
     {
-        $this->validate($request, [
-            'id'    => 'required|integer|min:1'
-        ]);
-        $pid = Auth::user()->profile->id;
-
-        $filter = UserFilter::whereFilterableType('App\Instance')
-            ->whereUserId($pid)
-            ->findOrFail($request->input('id'));
-        $filter->delete();
-        return redirect(route('settings.privacy.blocked-instances'));
+        // deprecated
+        abort(404);
     }
 
     public function blockedKeywords()

app/Http/Controllers/Stories/StoryApiV1Controller.php

@@ -20,339 +20,486 @@ use App\Jobs\StoryPipeline\StoryViewDeliver;
 use App\Services\AccountService;
 use App\Services\MediaPathService;
 use App\Services\StoryService;
+use App\Http\Resources\StoryView as StoryViewResource;
 
 class StoryApiV1Controller extends Controller
 {
-	public function carousel(Request $request)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-		$pid = $request->user()->profile_id;
+    const RECENT_KEY = 'pf:stories:recent-by-id:';
+    const RECENT_TTL = 300;
 
-		if(config('database.default') == 'pgsql') {
-			$s = Story::select('stories.*', 'followers.following_id')
-				->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-				->where('followers.profile_id', $pid)
-				->where('stories.active', true)
-				->get();
-		} else {
-			$s = Story::select('stories.*', 'followers.following_id')
-				->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-				->where('followers.profile_id', $pid)
-				->where('stories.active', true)
-				->orderBy('id')
-				->get();
-		}
+    public function carousel(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $pid = $request->user()->profile_id;
 
-		$nodes = $s->map(function($s) use($pid) {
-			$profile = AccountService::get($s->profile_id, true);
-			if(!$profile || !isset($profile['id'])) {
-				return false;
-			}
+        if(config('database.default') == 'pgsql') {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->map(function($s) {
+                        $r  = new \StdClass;
+                        $r->id = $s->id;
+                        $r->profile_id = $s->profile_id;
+                        $r->type = $s->type;
+                        $r->path = $s->path;
+                        return $r;
+                    })
+                    ->unique('profile_id');
+            });
+        } else {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->orderBy('id')
+                    ->get();
+            });
+        }
 
-			return [
-				'id' => (string) $s->id,
-				'pid' => (string) $s->profile_id,
-				'type' => $s->type,
-				'src' => url(Storage::url($s->path)),
-				'duration' => $s->duration ?? 3,
-				'seen' => StoryService::hasSeen($pid, $s->id),
-				'created_at' => $s->created_at->format('c')
-			];
-		})
-		->filter()
-		->groupBy('pid')
-		->map(function($item) use($pid) {
-			$profile = AccountService::get($item[0]['pid'], true);
-			$url = $profile['local'] ? url("/stories/{$profile['username']}") :
-				url("/i/rs/{$profile['id']}");
-			return [
-				'id' => 'pfs:' . $profile['id'],
-				'user' => [
-					'id' => (string) $profile['id'],
-					'username' => $profile['username'],
-					'username_acct' => $profile['acct'],
-					'avatar' => $profile['avatar'],
-					'local' => $profile['local'],
-					'is_author' => $profile['id'] == $pid
-				],
-				'nodes' => $item,
-				'url' => $url,
-				'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
-			];
-		})
-		->sortBy('seen')
-		->values();
+        $nodes = $s->map(function($s) use($pid) {
+            $profile = AccountService::get($s->profile_id, true);
+            if(!$profile || !isset($profile['id'])) {
+                return false;
+            }
 
-		$res = [
-			'self' => [],
-			'nodes' => $nodes,
-		];
+            return [
+                'id' => (string) $s->id,
+                'pid' => (string) $s->profile_id,
+                'type' => $s->type,
+                'src' => url(Storage::url($s->path)),
+                'duration' => $s->duration ?? 3,
+                'seen' => StoryService::hasSeen($pid, $s->id),
+                'created_at' => $s->created_at->format('c')
+            ];
+        })
+        ->filter()
+        ->groupBy('pid')
+        ->map(function($item) use($pid) {
+            $profile = AccountService::get($item[0]['pid'], true);
+            $url = $profile['local'] ? url("/stories/{$profile['username']}") :
+                url("/i/rs/{$profile['id']}");
+            return [
+                'id' => 'pfs:' . $profile['id'],
+                'user' => [
+                    'id' => (string) $profile['id'],
+                    'username' => $profile['username'],
+                    'username_acct' => $profile['acct'],
+                    'avatar' => $profile['avatar'],
+                    'local' => $profile['local'],
+                    'is_author' => $profile['id'] == $pid
+                ],
+                'nodes' => $item,
+                'url' => $url,
+                'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
+            ];
+        })
+        ->sortBy('seen')
+        ->values();
 
-		if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
-			$selfStories = Story::whereProfileId($pid)
-				->whereActive(true)
-				->get()
-				->map(function($s) use($pid) {
-					return [
-						'id' => (string) $s->id,
-						'type' => $s->type,
-						'src' => url(Storage::url($s->path)),
-						'duration' => $s->duration,
-						'seen' => true,
-						'created_at' => $s->created_at->format('c')
-					];
-				})
-				->sortBy('id')
-				->values();
-			$selfProfile = AccountService::get($pid, true);
-			$res['self'] = [
-				'user' => [
-					'id' => (string) $selfProfile['id'],
-					'username' => $selfProfile['acct'],
-					'avatar' => $selfProfile['avatar'],
-					'local' => $selfProfile['local'],
-					'is_author' => true
-				],
+        $res = [
+            'self' => [],
+            'nodes' => $nodes,
+        ];
 
-				'nodes' => $selfStories,
-			];
-		}
-		return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
-	}
+        if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
+            $selfStories = Story::whereProfileId($pid)
+                ->whereActive(true)
+                ->get()
+                ->map(function($s) use($pid) {
+                    return [
+                        'id' => (string) $s->id,
+                        'type' => $s->type,
+                        'src' => url(Storage::url($s->path)),
+                        'duration' => $s->duration,
+                        'seen' => true,
+                        'created_at' => $s->created_at->format('c')
+                    ];
+                })
+                ->sortBy('id')
+                ->values();
+            $selfProfile = AccountService::get($pid, true);
+            $res['self'] = [
+                'user' => [
+                    'id' => (string) $selfProfile['id'],
+                    'username' => $selfProfile['acct'],
+                    'avatar' => $selfProfile['avatar'],
+                    'local' => $selfProfile['local'],
+                    'is_author' => true
+                ],
 
-	public function add(Request $request)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+                'nodes' => $selfStories,
+            ];
+        }
+        return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+    }
 
-		$this->validate($request, [
-			'file' => function() {
-				return [
-					'required',
-					'mimetypes:image/jpeg,image/png,video/mp4',
-					'max:' . config_cache('pixelfed.max_photo_size'),
-				];
-			},
-			'duration' => 'sometimes|integer|min:0|max:30'
-		]);
+    public function selfCarousel(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $pid = $request->user()->profile_id;
 
-		$user = $request->user();
+        if(config('database.default') == 'pgsql') {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->map(function($s) {
+                        $r  = new \StdClass;
+                        $r->id = $s->id;
+                        $r->profile_id = $s->profile_id;
+                        $r->type = $s->type;
+                        $r->path = $s->path;
+                        return $r;
+                    })
+                    ->unique('profile_id');
+            });
+        } else {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->orderBy('id')
+                    ->get();
+            });
+        }
 
-		$count = Story::whereProfileId($user->profile_id)
-			->whereActive(true)
-			->where('expires_at', '>', now())
-			->count();
+        $nodes = $s->map(function($s) use($pid) {
+            $profile = AccountService::get($s->profile_id, true);
+            if(!$profile || !isset($profile['id'])) {
+                return false;
+            }
 
-		if($count >= Story::MAX_PER_DAY) {
-			abort(418, 'You have reached your limit for new Stories today.');
-		}
+            return [
+                'id' => (string) $s->id,
+                'pid' => (string) $s->profile_id,
+                'type' => $s->type,
+                'src' => url(Storage::url($s->path)),
+                'duration' => $s->duration ?? 3,
+                'seen' => StoryService::hasSeen($pid, $s->id),
+                'created_at' => $s->created_at->format('c')
+            ];
+        })
+        ->filter()
+        ->groupBy('pid')
+        ->map(function($item) use($pid) {
+            $profile = AccountService::get($item[0]['pid'], true);
+            $url = $profile['local'] ? url("/stories/{$profile['username']}") :
+                url("/i/rs/{$profile['id']}");
+            return [
+                'id' => 'pfs:' . $profile['id'],
+                'user' => [
+                    'id' => (string) $profile['id'],
+                    'username' => $profile['username'],
+                    'username_acct' => $profile['acct'],
+                    'avatar' => $profile['avatar'],
+                    'local' => $profile['local'],
+                    'is_author' => $profile['id'] == $pid
+                ],
+                'nodes' => $item,
+                'url' => $url,
+                'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
+            ];
+        })
+        ->sortBy('seen')
+        ->values();
 
-		$photo = $request->file('file');
-		$path = $this->storeMedia($photo, $user);
+        $selfProfile = AccountService::get($pid, true);
+        $res = [
+            'self' => [
+                'user' => [
+                    'id' => (string) $selfProfile['id'],
+                    'username' => $selfProfile['acct'],
+                    'avatar' => $selfProfile['avatar'],
+                    'local' => $selfProfile['local'],
+                    'is_author' => true
+                ],
 
-		$story = new Story();
-		$story->duration = $request->input('duration', 3);
-		$story->profile_id = $user->profile_id;
-		$story->type = Str::endsWith($photo->getMimeType(), 'mp4') ? 'video' :'photo';
-		$story->mime = $photo->getMimeType();
-		$story->path = $path;
-		$story->local = true;
-		$story->size = $photo->getSize();
-		$story->bearcap_token = str_random(64);
-		$story->expires_at = now()->addMinutes(1440);
-		$story->save();
+                'nodes' => [],
+            ],
+            'nodes' => $nodes,
+        ];
 
-		$url = $story->path;
+        if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
+            $selfStories = Story::whereProfileId($pid)
+                ->whereActive(true)
+                ->get()
+                ->map(function($s) use($pid) {
+                    return [
+                        'id' => (string) $s->id,
+                        'type' => $s->type,
+                        'src' => url(Storage::url($s->path)),
+                        'duration' => $s->duration,
+                        'seen' => true,
+                        'created_at' => $s->created_at->format('c')
+                    ];
+                })
+                ->sortBy('id')
+                ->values();
+            $res['self']['nodes'] = $selfStories;
+        }
+        return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+    }
 
-		$res = [
-			'code' => 200,
-			'msg'  => 'Successfully added',
-			'media_id' => (string) $story->id,
-			'media_url' => url(Storage::url($url)) . '?v=' . time(),
-			'media_type' => $story->type
-		];
+    public function add(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
 
-		return $res;
-	}
+        $this->validate($request, [
+            'file' => function() {
+                return [
+                    'required',
+                    'mimetypes:image/jpeg,image/png,video/mp4',
+                    'max:' . config_cache('pixelfed.max_photo_size'),
+                ];
+            },
+            'duration' => 'sometimes|integer|min:0|max:30'
+        ]);
 
-	public function publish(Request $request)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $user = $request->user();
 
-		$this->validate($request, [
-			'media_id' => 'required',
-			'duration' => 'required|integer|min:0|max:30',
-			'can_reply' => 'required|boolean',
-			'can_react' => 'required|boolean'
-		]);
+        $count = Story::whereProfileId($user->profile_id)
+            ->whereActive(true)
+            ->where('expires_at', '>', now())
+            ->count();
 
-		$id = $request->input('media_id');
-		$user = $request->user();
-		$story = Story::whereProfileId($user->profile_id)
-			->findOrFail($id);
+        if($count >= Story::MAX_PER_DAY) {
+            abort(418, 'You have reached your limit for new Stories today.');
+        }
 
-		$story->active = true;
-		$story->duration = $request->input('duration', 10);
-		$story->can_reply = $request->input('can_reply');
-		$story->can_react = $request->input('can_react');
-		$story->save();
+        $photo = $request->file('file');
+        $path = $this->storeMedia($photo, $user);
 
-		StoryService::delLatest($story->profile_id);
-		StoryFanout::dispatch($story)->onQueue('story');
-		StoryService::addRotateQueue($story->id);
+        $story = new Story();
+        $story->duration = $request->input('duration', 3);
+        $story->profile_id = $user->profile_id;
+        $story->type = Str::endsWith($photo->getMimeType(), 'mp4') ? 'video' :'photo';
+        $story->mime = $photo->getMimeType();
+        $story->path = $path;
+        $story->local = true;
+        $story->size = $photo->getSize();
+        $story->bearcap_token = str_random(64);
+        $story->expires_at = now()->addMinutes(1440);
+        $story->save();
 
-		return [
-			'code' => 200,
-			'msg'  => 'Successfully published',
-		];
-	}
+        $url = $story->path;
 
-	public function delete(Request $request, $id)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $res = [
+            'code' => 200,
+            'msg'  => 'Successfully added',
+            'media_id' => (string) $story->id,
+            'media_url' => url(Storage::url($url)) . '?v=' . time(),
+            'media_type' => $story->type
+        ];
 
-		$user = $request->user();
+        return $res;
+    }
 
-		$story = Story::whereProfileId($user->profile_id)
-			->findOrFail($id);
-		$story->active = false;
-		$story->save();
+    public function publish(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
 
-		StoryDelete::dispatch($story)->onQueue('story');
+        $this->validate($request, [
+            'media_id' => 'required',
+            'duration' => 'required|integer|min:0|max:30',
+            'can_reply' => 'required|boolean',
+            'can_react' => 'required|boolean'
+        ]);
 
-		return [
-			'code' => 200,
-			'msg'  => 'Successfully deleted'
-		];
-	}
+        $id = $request->input('media_id');
+        $user = $request->user();
+        $story = Story::whereProfileId($user->profile_id)
+            ->findOrFail($id);
 
-	public function viewed(Request $request)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $story->active = true;
+        $story->duration = $request->input('duration', 10);
+        $story->can_reply = $request->input('can_reply');
+        $story->can_react = $request->input('can_react');
+        $story->save();
 
-		$this->validate($request, [
-			'id'	=> 'required|min:1',
-		]);
-		$id = $request->input('id');
+        StoryService::delLatest($story->profile_id);
+        StoryFanout::dispatch($story)->onQueue('story');
+        StoryService::addRotateQueue($story->id);
 
-		$authed = $request->user()->profile;
+        return [
+            'code' => 200,
+            'msg'  => 'Successfully published',
+        ];
+    }
 
-		$story = Story::with('profile')
-			->findOrFail($id);
-		$exp = $story->expires_at;
+    public function delete(Request $request, $id)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
 
-		$profile = $story->profile;
+        $user = $request->user();
 
-		if($story->profile_id == $authed->id) {
-			return [];
-		}
+        $story = Story::whereProfileId($user->profile_id)
+            ->findOrFail($id);
+        $story->active = false;
+        $story->save();
 
-		$publicOnly = (bool) $profile->followedBy($authed);
-		abort_if(!$publicOnly, 403);
+        StoryDelete::dispatch($story)->onQueue('story');
 
-		$v = StoryView::firstOrCreate([
-			'story_id' => $id,
-			'profile_id' => $authed->id
-		]);
+        return [
+            'code' => 200,
+            'msg'  => 'Successfully deleted'
+        ];
+    }
 
-		if($v->wasRecentlyCreated) {
-			Story::findOrFail($story->id)->increment('view_count');
+    public function viewed(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
 
-			if($story->local == false) {
-				StoryViewDeliver::dispatch($story, $authed)->onQueue('story');
-			}
-		}
+        $this->validate($request, [
+            'id'    => 'required|min:1',
+        ]);
+        $id = $request->input('id');
 
-		Cache::forget('stories:recent:by_id:' . $authed->id);
-		StoryService::addSeen($authed->id, $story->id);
-		return ['code' => 200];
-	}
+        $authed = $request->user()->profile;
 
-	public function comment(Request $request)
-	{
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-		$this->validate($request, [
-			'sid' => 'required',
-			'caption' => 'required|string'
-		]);
-		$pid = $request->user()->profile_id;
-		$text = $request->input('caption');
+        $story = Story::with('profile')
+            ->findOrFail($id);
+        $exp = $story->expires_at;
 
-		$story = Story::findOrFail($request->input('sid'));
+        $profile = $story->profile;
 
-		abort_if(!$story->can_reply, 422);
+        if($story->profile_id == $authed->id) {
+            return [];
+        }
 
-		$status = new Status;
-		$status->type = 'story:reply';
-		$status->profile_id = $pid;
-		$status->caption = $text;
-		$status->rendered = $text;
-		$status->scope = 'direct';
-		$status->visibility = 'direct';
-		$status->in_reply_to_profile_id = $story->profile_id;
-		$status->entities = json_encode([
-			'story_id' => $story->id
-		]);
-		$status->save();
+        $publicOnly = (bool) $profile->followedBy($authed);
+        abort_if(!$publicOnly, 403);
 
-		$dm = new DirectMessage;
-		$dm->to_id = $story->profile_id;
-		$dm->from_id = $pid;
-		$dm->type = 'story:comment';
-		$dm->status_id = $status->id;
-		$dm->meta = json_encode([
-			'story_username' => $story->profile->username,
-			'story_actor_username' => $request->user()->username,
-			'story_id' => $story->id,
-			'story_media_url' => url(Storage::url($story->path)),
-			'caption' => $text
-		]);
-		$dm->save();
+        $v = StoryView::firstOrCreate([
+            'story_id' => $id,
+            'profile_id' => $authed->id
+        ]);
 
-		Conversation::updateOrInsert(
-			[
-				'to_id' => $story->profile_id,
-				'from_id' => $pid
-			],
-			[
-				'type' => 'story:comment',
-				'status_id' => $status->id,
-				'dm_id' => $dm->id,
-				'is_hidden' => false
-			]
-		);
+        if($v->wasRecentlyCreated) {
+            Story::findOrFail($story->id)->increment('view_count');
 
-		if($story->local) {
-			$n = new Notification;
-			$n->profile_id = $dm->to_id;
-			$n->actor_id = $dm->from_id;
-			$n->item_id = $dm->id;
-			$n->item_type = 'App\DirectMessage';
-			$n->action = 'story:comment';
-			$n->save();
-		} else {
-			StoryReplyDeliver::dispatch($story, $status)->onQueue('story');
-		}
+            if($story->local == false) {
+                StoryViewDeliver::dispatch($story, $authed)->onQueue('story');
+            }
+        }
 
-		return [
-			'code' => 200,
-			'msg'  => 'Sent!'
-		];
-	}
+        Cache::forget('stories:recent:by_id:' . $authed->id);
+        StoryService::addSeen($authed->id, $story->id);
+        return ['code' => 200];
+    }
 
-	protected function storeMedia($photo, $user)
-	{
-		$mimes = explode(',', config_cache('pixelfed.media_types'));
-		if(in_array($photo->getMimeType(), [
-			'image/jpeg',
-			'image/png',
-			'video/mp4'
-		]) == false) {
-			abort(400, 'Invalid media type');
-			return;
-		}
+    public function comment(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $this->validate($request, [
+            'sid' => 'required',
+            'caption' => 'required|string'
+        ]);
+        $pid = $request->user()->profile_id;
+        $text = $request->input('caption');
 
-		$storagePath = MediaPathService::story($user->profile);
-		$path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
-		return $path;
-	}
+        $story = Story::findOrFail($request->input('sid'));
+
+        abort_if(!$story->can_reply, 422);
+
+        $status = new Status;
+        $status->type = 'story:reply';
+        $status->profile_id = $pid;
+        $status->caption = $text;
+        $status->rendered = $text;
+        $status->scope = 'direct';
+        $status->visibility = 'direct';
+        $status->in_reply_to_profile_id = $story->profile_id;
+        $status->entities = json_encode([
+            'story_id' => $story->id
+        ]);
+        $status->save();
+
+        $dm = new DirectMessage;
+        $dm->to_id = $story->profile_id;
+        $dm->from_id = $pid;
+        $dm->type = 'story:comment';
+        $dm->status_id = $status->id;
+        $dm->meta = json_encode([
+            'story_username' => $story->profile->username,
+            'story_actor_username' => $request->user()->username,
+            'story_id' => $story->id,
+            'story_media_url' => url(Storage::url($story->path)),
+            'caption' => $text
+        ]);
+        $dm->save();
+
+        Conversation::updateOrInsert(
+            [
+                'to_id' => $story->profile_id,
+                'from_id' => $pid
+            ],
+            [
+                'type' => 'story:comment',
+                'status_id' => $status->id,
+                'dm_id' => $dm->id,
+                'is_hidden' => false
+            ]
+        );
+
+        if($story->local) {
+            $n = new Notification;
+            $n->profile_id = $dm->to_id;
+            $n->actor_id = $dm->from_id;
+            $n->item_id = $dm->id;
+            $n->item_type = 'App\DirectMessage';
+            $n->action = 'story:comment';
+            $n->save();
+        } else {
+            StoryReplyDeliver::dispatch($story, $status)->onQueue('story');
+        }
+
+        return [
+            'code' => 200,
+            'msg'  => 'Sent!'
+        ];
+    }
+
+    protected function storeMedia($photo, $user)
+    {
+        $mimes = explode(',', config_cache('pixelfed.media_types'));
+        if(in_array($photo->getMimeType(), [
+            'image/jpeg',
+            'image/png',
+            'video/mp4'
+        ]) == false) {
+            abort(400, 'Invalid media type');
+            return;
+        }
+
+        $storagePath = MediaPathService::story($user->profile);
+        $path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
+        return $path;
+    }
+
+    public function viewers(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
+        $this->validate($request, [
+            'sid' => 'required|string|min:1|max:50'
+        ]);
+
+        $pid = $request->user()->profile_id;
+        $sid = $request->input('sid');
+
+        $story = Story::whereProfileId($pid)
+            ->whereActive(true)
+            ->findOrFail($sid);
+
+        $viewers = StoryView::whereStoryId($story->id)
+            ->orderByDesc('id')
+            ->cursorPaginate(10);
+
+        return StoryViewResource::collection($viewers);
+    }
 }

app/Http/Resources/StoryView.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Request;
+use Illuminate\Http\Resources\Json\JsonResource;
+use App\Services\AccountService;
+
+class StoryView extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @return array<string, mixed>
+     */
+    public function toArray(Request $request)
+    {
+        return AccountService::get($this->profile_id, true);
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+
+class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        AvatarService::cleanup($this->avatar, true);
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+use Illuminate\Support\Str;
+
+class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $avatar = $this->avatar;
+
+        $disk = AvatarService::disk();
+
+        $files = collect(AvatarService::storage($avatar));
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/CreateAvatar.php

@@ -2,19 +2,25 @@
 
 namespace App\Jobs\AvatarPipeline;
 
-use App\Avatar;
-use App\Profile;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Avatar;
+use App\Profile;
 
-class CreateAvatar implements ShouldQueue
+class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-    protected $profile;
+    public $profile;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
 
     /**
      * Delete the job if its models no longer exist.
@@ -22,6 +28,31 @@ class CreateAvatar implements ShouldQueue
      * @var bool
      */
     public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:create:' . $this->profile->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
+    }
     
     /**
      * Create a new job instance.
@@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
      */
     public function __construct(Profile $profile)
     {
-        $this->profile = $profile;
+        $this->profile = $profile->withoutRelations();
     }
 
     /**
@@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
     public function handle()
     {
         $profile = $this->profile;
+        $isRemote = (bool) $profile->private_key == null;
         $path = 'public/avatars/default.jpg';
-        $avatar = new Avatar();
-        $avatar->profile_id = $profile->id;
-        $avatar->media_path = $path;
-        $avatar->change_count = 0;
-        $avatar->last_processed_at = \Carbon\Carbon::now();
-        $avatar->save();
+        Avatar::updateOrCreate(
+            [
+                'profile_id' => $profile->id,
+            ],
+            [
+                'media_path' => $path,
+                'change_count' => 0,
+                'is_remote' => $isRemote,
+                'last_processed_at' => now()
+            ]
+        );
     }
 }

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
+		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;
 	}

app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

@@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
 			$avatar->save();
 		}
 
-
 		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;

app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php

@@ -57,7 +57,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue
         $status = $this->status;
 
         if(AccountService::get($status->profile_id, true)) {
-            DecrementPostCount::dispatch($status->profile_id)->onQueue('feed');
+            DecrementPostCount::dispatch($status->profile_id)->onQueue('low');
         }
 
         NetworkTimelineService::del($status->id);

app/Jobs/DirectPipeline/DirectDeletePipeline.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Jobs\DirectPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Util\ActivityPub\Helpers;
+
+class DirectDeletePipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+
+    protected $profile;
+    protected $url;
+    protected $payload;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($profile, $url, $payload)
+    {
+        $this->profile = $profile;
+        $this->url = $url;
+        $this->payload = $payload;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
+    }
+}

app/Jobs/DirectPipeline/DirectDeliverPipeline.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Jobs\DirectPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Util\ActivityPub\Helpers;
+
+class DirectDeliverPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+
+    protected $profile;
+    protected $url;
+    protected $payload;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($profile, $url, $payload)
+    {
+        $this->profile = $profile;
+        $this->url = $url;
+        $this->payload = $payload;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
+    }
+}

app/Jobs/FollowPipeline/FollowServiceWarmCache.php

@@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use Cache;
 use DB;
+use Storage;
+use App\Follower;
 use App\Profile;
 
 class FollowServiceWarmCache implements ShouldQueue
@@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue
 	public $timeout = 5000;
 	public $failOnTimeout = false;
 
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->profileId))->dontRelease()];
+    }
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue
 	{
 		$id = $this->profileId;
 
+        if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) {
+            return;
+        }
+
 		$account = AccountService::get($id, true);
 
 		if(!$account) {
@@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue
 			return;
 		}
 
-		DB::table('followers')
-			->select('id', 'following_id', 'profile_id')
-			->whereFollowingId($id)
-			->orderBy('id')
-			->chunk(200, function($followers) use($id) {
-			foreach($followers as $follow) {
-				FollowerService::add($follow->profile_id, $id);
-			}
-		});
+        $hasFollowerPostProcessing = false;
+        $hasFollowingPostProcessing = false;
 
-		DB::table('followers')
-			->select('id', 'following_id', 'profile_id')
-			->whereProfileId($id)
-			->orderBy('id')
-			->chunk(200, function($followers) use($id) {
-			foreach($followers as $follow) {
-				FollowerService::add($id, $follow->following_id);
-			}
-		});
+        if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
+            $following = [];
+            $followers = [];
+    		foreach(Follower::where('following_id', $id)->orWhere('profile_id', $id)->lazyById(500) as $follow) {
+                if($follow->following_id != $id && $follow->profile_id != $id) {
+                    continue;
+                }
+                if($follow->profile_id == $id) {
+                    $following[] = $follow->following_id;
+                } else {
+                    $followers[] = $follow->profile_id;
+                }
+            }
+
+            if(count($followers) > 100) {
+                // store follower ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers));
+                $hasFollowerPostProcessing = true;
+            } else {
+                foreach($followers as $follower) {
+                    FollowerService::add($follower, $id);
+                }
+            }
+
+            if(count($following) > 100) {
+                // store following ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following));
+                $hasFollowingPostProcessing = true;
+            } else {
+                foreach($following as $following) {
+                    FollowerService::add($id, $following);
+                }
+            }
+        }
 
 		Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800);
 		Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800);
@@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue
 
 		AccountService::del($id);
 
+        if($hasFollowingPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow');
+        }
+
+        if($hasFollowerPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow');
+        }
+
 		return;
 	}
 }

app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Jobs\FollowPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\AccountService;
+use App\Services\FollowerService;
+use Cache;
+use DB;
+use Storage;
+use App\Follower;
+use App\Profile;
+
+class FollowServiceWarmCacheLargeIngestPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $profileId;
+    public $followType;
+    public $tries = 5;
+    public $timeout = 5000;
+    public $failOnTimeout = false;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($profileId, $followType = 'following')
+    {
+        $this->profileId = $profileId;
+        $this->followType = $followType;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $pid = $this->profileId;
+        $type = $this->followType;
+
+        if($type === 'followers') {
+            $key = 'follow-warm-cache/' . $pid . '/followers.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($id, $pid, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        if($type === 'following') {
+            $key = 'follow-warm-cache/' . $pid . '/following.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($pid, $id, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        sleep(random_int(2, 5));
+        $files = Storage::files('follow-warm-cache/' . $pid);
+        if(empty($files)) {
+            Storage::deleteDirectory('follow-warm-cache/' . $pid);
+        }
+    }
+}

app/Jobs/FollowPipeline/UnfollowPipeline.php

@@ -17,6 +17,7 @@ use Illuminate\Support\Facades\Redis;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use App\Services\NotificationService;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UnfollowPipeline implements ShouldQueue
 {
@@ -55,6 +56,8 @@ class UnfollowPipeline implements ShouldQueue
 			return;
 		}
 
+		FeedUnfollowPipeline::dispatch($actor, $target)->onQueue('follow');
+
 		FollowerService::remove($actor, $target);
 
 		$actorProfileSync = Cache::get(FollowerService::FOLLOWING_SYNC_KEY . $actor);

app/Jobs/HomeFeedPipeline/FeedFollowPipeline.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\HomeTimelineService;
+use App\Services\SnowflakeService;
+use App\Status;
+
+class FeedFollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $minId = SnowflakeService::byDate(now()->subWeeks(6));
+
+        $ids = Status::where('id', '>', $minId)
+            ->where('profile_id', $followingId)
+            ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+            ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+            ->whereIn('visibility',['public', 'unlisted', 'private'])
+            ->orderByDesc('id')
+            ->limit(HomeTimelineService::FOLLOWER_FEED_POST_LIMIT)
+            ->pluck('id');
+
+        foreach($ids as $id) {
+            HomeTimelineService::add($actorId, $id);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        HomeTimelineService::add($this->pid, $this->sid);
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')
+            ->whereFilterableId($status['account']['id'])
+            ->whereIn('filter_type', ['mute', 'block'])
+            ->pluck('user_id')
+            ->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php

@@ -0,0 +1,112 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')
+            ->whereFilterableId($status['account']['id'])
+            ->whereIn('filter_type', ['mute', 'block'])
+            ->pluck('user_id')
+            ->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php

@@ -0,0 +1,98 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveDomainPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if(!config('exp.cached_home_timeline')) {
+            return;
+        }
+
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        if(!$this->pid || !$this->domain) {
+            return;
+        }
+        $domain = strtolower($this->domain);
+        $pid = $this->pid;
+        $posts = HomeTimelineService::get($pid, '0', '-1');
+
+        foreach($posts as $post) {
+            $status = StatusService::get($post, false);
+            if(!$status || !isset($status['url'])) {
+                HomeTimelineService::rem($pid, $post);
+                continue;
+            }
+            $host = strtolower(parse_url($status['url'], PHP_URL_HOST));
+            if($host === strtolower(config('pixelfed.domain.app')) || !$host) {
+                continue;
+            }
+            if($host === $domain) {
+                HomeTimelineService::rem($pid, $status['id']);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemovePipeline.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemovePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        HomeTimelineService::rem($this->pid, $this->sid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveRemotePipeline.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedUnfollowPipeline.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedUnfollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $ids = HomeTimelineService::get($actorId, 0, -1);
+        foreach($ids as $id) {
+            $status = StatusService::get($id, false);
+            if($status && isset($status['account'], $status['account']['id'])) {
+                if($status['account']['id'] == $followingId) {
+                    HomeTimelineService::rem($actorId, $id);
+                }
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedWarmCachePipeline.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\HomeTimelineService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class FeedWarmCachePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:warm-cache:pid:' . $this->pid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:warm-cache:pid:{$this->pid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid)
+    {
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $pid = $this->pid;
+        HomeTimelineService::warmCache($pid, true, 400, true);
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $hashtag;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:insert:' . $this->hashtag->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:insert:{$this->hashtag->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(StatusHashtag $hashtag)
+    {
+        $this->hashtag = $hashtag;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hashtag = $this->hashtag;
+        $sid = $hashtag->status_id;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $hashtag->status_id);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php

@@ -0,0 +1,92 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $hid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:remove:' . $this->hid . ':' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:remove:{$this->hid}:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $hid)
+    {
+        $this->sid = $sid;
+        $this->hid = $hid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $hid = $this->hid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = HashtagFollowService::getPidByHid($hid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Support\Facades\Cache;
+use App\Follower;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class HashtagUnfollowPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $hid;
+    protected $slug;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($hid, $pid, $slug)
+    {
+        $this->hid = $hid;
+        $this->pid = $pid;
+        $this->slug = $slug;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hid = $this->hid;
+        $pid = $this->pid;
+        $slug = strtolower($this->slug);
+
+        $statusIds = HomeTimelineService::get($pid, 0, -1);
+
+        $followingIds = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
+            $following = Follower::whereProfileId($pid)->pluck('following_id');
+            return $following->push($pid)->toArray();
+        });
+
+        foreach($statusIds as $id) {
+            $status = StatusService::get($id, false);
+            if(!$status || empty($status['tags'])) {
+                HomeTimelineService::rem($pid, $id);
+                continue;
+            }
+            $following = in_array((int) $status['account']['id'], $followingIds);
+            if($following === true) {
+                continue;
+            }
+
+            $tags = collect($status['tags'])->map(function($tag) {
+                return strtolower($tag['name']);
+            })->filter()->values()->toArray();
+
+            if(in_array($slug, $tags)) {
+                HomeTimelineService::rem($pid, $id);
+            }
+        }
+    }
+}

app/Jobs/ImageOptimizePipeline/ImageResize.php

@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Log;
 
 class ImageResize implements ShouldQueue
 {
@@ -46,6 +47,7 @@ class ImageResize implements ShouldQueue
         }
         $path = storage_path('app/'.$media->media_path);
         if (!is_file($path) || $media->skip_optimize) {
+            Log::info('Tried to optimize media that does not exist or is not readable. ' . $path);
             return;
         }
 
@@ -57,6 +59,7 @@ class ImageResize implements ShouldQueue
             $img = new Image();
             $img->resizeImage($media);
         } catch (Exception $e) {
+            Log::error($e);
         }
 
         ImageThumbnail::dispatch($media)->onQueue('mmo');

app/Jobs/InboxPipeline/InboxValidator.php

@@ -193,7 +193,7 @@ class InboxValidator implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InboxPipeline/InboxWorker.php

@@ -173,7 +173,7 @@ class InboxWorker implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Jobs\InternalPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Notification;
+use Cache;
+use App\Services\NotificationService;
+
+class NotificationEpochUpdatePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 1500;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'ip:notification-epoch-update';
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping('ip:notification-epoch-update'))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $rec = Notification::where('created_at', '>', now()->subMonths(6))->first();
+        $id = 1;
+        if($rec) {
+            $id = $rec->id;
+        }
+        Cache::put(NotificationService::EPOCH_CACHE_KEY . '6', $id, 1209600);
+    }
+}

app/Jobs/MediaPipeline/MediaDeletePipeline.php

@@ -10,8 +10,11 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
+use App\Services\Media\MediaHlsService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class MediaDeletePipeline implements ShouldQueue
+class MediaDeletePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -20,8 +23,34 @@ class MediaDeletePipeline implements ShouldQueue
     public $timeout = 300;
     public $tries = 3;
     public $maxExceptions = 1;
+    public $failOnTimeout = true;
     public $deleteWhenMissingModels = true;
 
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:purge-job:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:purge-job:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
 	public function __construct(Media $media)
 	{
 		$this->media = $media;
@@ -63,9 +92,17 @@ class MediaDeletePipeline implements ShouldQueue
 			$disk->delete($thumb);
 		}
 
+		if($media->hls_path != null) {
+            $files = MediaHlsService::allFiles($media);
+            if($files && count($files)) {
+                foreach($files as $file) {
+                    $disk->delete($file);
+                }
+            }
+		}
+
 		$media->delete();
 
 		return 1;
 	}
-
 }

app/Jobs/ProfilePipeline/DecrementPostCount.php

@@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue
             return 1;
         }
 
-        if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-            $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-            $profile->save();
-            AccountService::del($id);
-        } else {
-            $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0;
-            $profile->save();
-            AccountService::del($id);
-        }
+        $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0;
+        $profile->save();
+        AccountService::del($id);
 
         return 1;
     }

app/Jobs/ProfilePipeline/IncrementPostCount.php

@@ -8,16 +8,48 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use App\Profile;
 use App\Status;
 use App\Services\AccountService;
 
-class IncrementPostCount implements ShouldQueue
+class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
 	public $id;
 
+	public $timeout = 900;
+	public $tries = 3;
+	public $maxExceptions = 1;
+	public $failOnTimeout = true;
+
+	/**
+	 * The number of seconds after which the job's unique lock will be released.
+	 *
+	 * @var int
+	 */
+	public $uniqueFor = 3600;
+
+	/**
+	 * Get the unique ID for the job.
+	 */
+	public function uniqueId(): string
+	{
+		return 'propipe:ipc:' . $this->id;
+	}
+
+	/**
+	 * Get the middleware the job should pass through.
+	 *
+	 * @return array<int, object>
+	 */
+	public function middleware(): array
+	{
+		return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()];
+	}
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -43,17 +75,11 @@ class IncrementPostCount implements ShouldQueue
 			return 1;
 		}
 
-		if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-			$profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-			$profile->last_status_at = now();
-			$profile->save();
-			AccountService::del($id);
-		} else {
-			$profile->status_count = $profile->status_count + 1;
-			$profile->last_status_at = now();
-			$profile->save();
-			AccountService::del($id);
-		}
+		$profile->status_count = $profile->status_count + 1;
+		$profile->last_status_at = now();
+		$profile->save();
+		AccountService::del($id);
+		AccountService::get($id);
 
 		return 1;
 	}

app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php

@@ -0,0 +1,119 @@
+<?php
+
+namespace App\Jobs\ProfilePipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Follower;
+use App\Profile;
+use App\Notification;
+use DB;
+use App\Services\AccountService;
+use App\Services\FollowerService;
+use App\Services\NotificationService;
+
+class ProfilePurgeFollowersByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'followers:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("followers:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        $pid = $this->pid;
+        $domain = $this->domain;
+
+        $query = 'SELECT f.*
+            FROM followers f
+            JOIN profiles p ON p.id = f.profile_id OR p.id = f.following_id
+            WHERE (f.profile_id = ? OR f.following_id = ?)
+            AND p.domain = ?;';
+        $params = [$pid, $pid, $domain];
+
+        foreach(DB::cursor($query, $params) as $n) {
+            if(!$n || !$n->id) {
+                continue;
+            }
+            $follower = Follower::find($n->id);
+            if($follower->following_id == $pid && $follower->profile_id) {
+                FollowerService::remove($follower->profile_id, $pid, true);
+                $follower->delete();
+            } else if ($follower->profile_id == $pid && $follower->following_id) {
+                FollowerService::remove($follower->following_id, $pid, true);
+                $follower->delete();
+            }
+        }
+
+        $profile = Profile::find($pid);
+
+        $followerCount = DB::table('profiles')
+            ->join('followers', 'profiles.id', '=', 'followers.following_id')
+            ->where('followers.following_id', $pid)
+            ->count();
+
+        $followingCount = DB::table('profiles')
+            ->join('followers', 'profiles.id', '=', 'followers.following_id')
+            ->where('followers.profile_id', $pid)
+            ->count();
+
+        $profile->followers_count = $followerCount;
+        $profile->following_count = $followingCount;
+        $profile->save();
+
+        AccountService::del($profile->id);
+    }
+}

app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Jobs\ProfilePipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Notification;
+use DB;
+use App\Services\NotificationService;
+
+class ProfilePurgeNotificationsByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'notify:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("notify:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        $pid = $this->pid;
+        $domain = $this->domain;
+
+        $query = 'SELECT notifications.*
+            FROM profiles
+            JOIN notifications on profiles.id = notifications.actor_id
+            WHERE notifications.profile_id = ?
+            AND profiles.domain = ?';
+        $params = [$pid, $domain];
+
+        foreach(DB::cursor($query, $params) as $n) {
+            if(!$n || !$n->id) {
+                continue;
+            }
+            Notification::where('id', $n->id)->delete();
+            NotificationService::del($pid, $n->id);
+        }
+    }
+}

app/Jobs/SharePipeline/SharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
 
 class SharePipeline implements ShouldQueue
 {
@@ -82,6 +83,8 @@ class SharePipeline implements ShouldQueue
 			]
 		);
 
+		FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		return $this->remoteAnnounceDeliver();
 	}
 

app/Jobs/SharePipeline/UndoSharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
 
 class UndoSharePipeline implements ShouldQueue
 {
@@ -35,6 +36,8 @@ class UndoSharePipeline implements ShouldQueue
 		$actor = $status->profile;
 		$parent = Status::find($status->reblog_of_id);
 
+		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		if($parent) {
 			$target = $parent->profile_id;
 			ReblogService::removePostReblog($parent->profile_id, $status->id);

app/Jobs/StatusPipeline/RemoteStatusDelete.php

@@ -21,9 +21,11 @@ use App\{
 };
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use League\Fractal;
 use Illuminate\Support\Str;
 use League\Fractal\Serializer\ArraySerializer;
@@ -37,8 +39,10 @@ use App\Services\AccountService;
 use App\Services\CollectionService;
 use App\Services\StatusService;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
+use App\Jobs\ProfilePipeline\DecrementPostCount;
+use App\Services\NotificationService;
 
-class RemoteStatusDelete implements ShouldQueue
+class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -51,9 +55,35 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public $deleteWhenMissingModels = true;
 
-    public $timeout = 90;
-    public $tries = 2;
-    public $maxExceptions = 1;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 180;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'status:remote:delete:' . $this->status->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()];
+    }
 
     /**
      * Create a new job instance.
@@ -62,7 +92,7 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public function __construct(Status $status)
     {
-        $this->status = $status;
+        $this->status = $status->withoutRelations();
     }
 
     /**
@@ -77,14 +107,10 @@ class RemoteStatusDelete implements ShouldQueue
         if($status->deleted_at) {
             return;
         }
-        $profile = $this->status->profile;
 
         StatusService::del($status->id, true);
 
-        if($profile->status_count && $profile->status_count > 0) {
-            $profile->status_count = $profile->status_count - 1;
-            $profile->save();
-        }
+        DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox');
 
         return $this->unlinkRemoveMedia($status);
     }
@@ -112,14 +138,34 @@ class RemoteStatusDelete implements ShouldQueue
                 CollectionService::removeItem($col->collection_id, $col->object_id);
                 $col->delete();
         });
-        DirectMessage::whereStatusId($status->id)->delete();
+        $dms = DirectMessage::whereStatusId($status->id)->get();
+        foreach($dms as $dm) {
+            $not = Notification::whereItemType('App\DirectMessage')
+                ->whereItemId($dm->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $dm->delete();
+        }
         Like::whereStatusId($status->id)->forceDelete();
         Media::whereStatusId($status->id)
         ->get()
         ->each(function($media) {
             MediaDeletePipeline::dispatch($media)->onQueue('mmo');
         });
-        MediaTag::where('status_id', $status->id)->delete();
+        $mediaTags = MediaTag::where('status_id', $status->id)->get();
+        foreach($mediaTags as $mtag) {
+            $not = Notification::whereItemType('App\MediaTag')
+                ->whereItemId($mtag->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $mtag->delete();
+        }
         Mention::whereStatusId($status->id)->forceDelete();
         Notification::whereItemType('App\Status')
             ->whereItemId($status->id)

app/Jobs/StatusPipeline/StatusDelete.php

@@ -35,6 +35,7 @@ use GuzzleHttp\Promise;
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\CollectionService;
 use App\Services\StatusService;
+use App\Services\NotificationService;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 
 class StatusDelete implements ShouldQueue
@@ -115,10 +116,30 @@ class StatusDelete implements ShouldQueue
                 $col->delete();
         });
 
-        DirectMessage::whereStatusId($status->id)->delete();
+        $dms = DirectMessage::whereStatusId($status->id)->get();
+        foreach($dms as $dm) {
+            $not = Notification::whereItemType('App\DirectMessage')
+                ->whereItemId($dm->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $dm->delete();
+        }
         Like::whereStatusId($status->id)->delete();
 
-		MediaTag::where('status_id', $status->id)->delete();
+        $mediaTags = MediaTag::where('status_id', $status->id)->get();
+        foreach($mediaTags as $mtag) {
+            $not = Notification::whereItemType('App\MediaTag')
+                ->whereItemId($mtag->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $mtag->delete();
+        }
         Mention::whereStatusId($status->id)->forceDelete();
 
 		Notification::whereItemType('App\Status')

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -19,168 +19,189 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use App\Services\StatusService;
 use App\Services\UserFilterService;
+use App\Services\AdminShadowFilterService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
 
 class StatusEntityLexer implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-	protected $status;
-	protected $entities;
-	protected $autolink;
+    protected $status;
+    protected $entities;
+    protected $autolink;
 
-	/**
-	 * Delete the job if its models no longer exist.
-	 *
-	 * @var bool
-	 */
-	public $deleteWhenMissingModels = true;
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;
 
-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct(Status $status)
-	{
-		$this->status = $status;
-	}
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Status $status)
+    {
+        $this->status = $status;
+    }
 
-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$profile = $this->status->profile;
-		$status = $this->status;
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $profile = $this->status->profile;
+        $status = $this->status;
 
-		if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
-			$profile->status_count = $profile->status_count + 1;
-			$profile->save();
-		}
+        if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            $profile->status_count = $profile->status_count + 1;
+            $profile->save();
+        }
 
-		if($profile->no_autolink == false) {
-			$this->parseEntities();
-		}
-	}
+        if($profile->no_autolink == false) {
+            $this->parseEntities();
+        }
+    }
 
-	public function parseEntities()
-	{
-		$this->extractEntities();
-	}
+    public function parseEntities()
+    {
+        $this->extractEntities();
+    }
 
-	public function extractEntities()
-	{
-		$this->entities = Extractor::create()->extract($this->status->caption);
-		$this->autolinkStatus();
-	}
+    public function extractEntities()
+    {
+        $this->entities = Extractor::create()->extract($this->status->caption);
+        $this->autolinkStatus();
+    }
 
-	public function autolinkStatus()
-	{
-		$this->autolink = Autolink::create()->autolink($this->status->caption);
-		$this->storeEntities();
-	}
+    public function autolinkStatus()
+    {
+        $this->autolink = Autolink::create()->autolink($this->status->caption);
+        $this->storeEntities();
+    }
 
-	public function storeEntities()
-	{
-		$this->storeHashtags();
-		DB::transaction(function () {
-			$status = $this->status;
-			$status->rendered = nl2br($this->autolink);
-			$status->save();
-		});
-	}
+    public function storeEntities()
+    {
+        $this->storeHashtags();
+        DB::transaction(function () {
+            $status = $this->status;
+            $status->rendered = nl2br($this->autolink);
+            $status->save();
+        });
+    }
 
-	public function storeHashtags()
-	{
-		$tags = array_unique($this->entities['hashtags']);
-		$status = $this->status;
+    public function storeHashtags()
+    {
+        $tags = array_unique($this->entities['hashtags']);
+        $status = $this->status;
 
-		foreach ($tags as $tag) {
-			if(mb_strlen($tag) > 124) {
-				continue;
-			}
-			DB::transaction(function () use ($status, $tag) {
-				$slug = str_slug($tag, '-', false);
-				$hashtag = Hashtag::where('slug', $slug)->first();
-				if (!$hashtag) {
-					$hashtag = Hashtag::create(
-						['name' => $tag, 'slug' => $slug]
-					);
-				}
+        foreach ($tags as $tag) {
+            if(mb_strlen($tag) > 124) {
+                continue;
+            }
+            DB::transaction(function () use ($status, $tag) {
+                $slug = str_slug($tag, '-', false);
 
-				StatusHashtag::firstOrCreate(
-					[
-						'status_id' => $status->id,
-						'hashtag_id' => $hashtag->id,
-						'profile_id' => $status->profile_id,
-						'status_visibility' => $status->visibility,
-					]
-				);
-			});
-		}
-		$this->storeMentions();
-	}
+                $hashtag = Hashtag::firstOrCreate([
+                    'slug' => $slug
+                ], [
+                    'name' => $tag
+                ]);
 
-	public function storeMentions()
-	{
-		$mentions = array_unique($this->entities['mentions']);
-		$status = $this->status;
+                StatusHashtag::firstOrCreate(
+                    [
+                        'status_id' => $status->id,
+                        'hashtag_id' => $hashtag->id,
+                        'profile_id' => $status->profile_id,
+                        'status_visibility' => $status->visibility,
+                    ]
+                );
+            });
+        }
+        $this->storeMentions();
+    }
 
-		foreach ($mentions as $mention) {
-			$mentioned = Profile::whereUsername($mention)->first();
+    public function storeMentions()
+    {
+        $mentions = array_unique($this->entities['mentions']);
+        $status = $this->status;
 
-			if (empty($mentioned) || !isset($mentioned->id)) {
-				continue;
-			}
+        foreach ($mentions as $mention) {
+            $mentioned = Profile::whereUsername($mention)->first();
+
+            if (empty($mentioned) || !isset($mentioned->id)) {
+                continue;
+            }
             $blocks = UserFilterService::blocks($mentioned->id);
             if($blocks && in_array($status->profile_id, $blocks)) {
                 continue;
             }
 
-			DB::transaction(function () use ($status, $mentioned) {
-				$m = new Mention();
-				$m->status_id = $status->id;
-				$m->profile_id = $mentioned->id;
-				$m->save();
+            DB::transaction(function () use ($status, $mentioned) {
+                $m = new Mention();
+                $m->status_id = $status->id;
+                $m->profile_id = $mentioned->id;
+                $m->save();
 
-				MentionPipeline::dispatch($status, $m);
-			});
-		}
-		$this->deliver();
-	}
+                MentionPipeline::dispatch($status, $m);
+            });
+        }
+        $this->fanout();
+    }
 
-	public function deliver()
-	{
-		$status = $this->status;
-		$types = [
-			'photo',
-			'photo:album',
-			'video',
-			'video:album',
-			'photo:video:album'
-		];
+    public function fanout()
+    {
+        $status = $this->status;
+        StatusService::refresh($status->id);
 
-		if(config_cache('pixelfed.bouncer.enabled')) {
-			Bouncer::get($status);
-		}
+        if(config('exp.cached_home_timeline')) {
+            if( $status->in_reply_to_id === null &&
+                in_array($status->scope, ['public', 'unlisted', 'private'])
+            ) {
+                FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+            }
+        }
+        $this->deliver();
+    }
 
-		Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
-		$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
-		if( $status->uri == null &&
-			$status->scope == 'public' &&
-			in_array($status->type, $types) &&
-			$status->in_reply_to_id === null &&
-			$status->reblog_of_id === null &&
-			($hideNsfw ? $status->is_nsfw == false : true)
-		) {
-			PublicTimelineService::add($status->id);
-		}
+    public function deliver()
+    {
+        $status = $this->status;
+        $types = [
+            'photo',
+            'photo:album',
+            'video',
+            'video:album',
+            'photo:video:album'
+        ];
 
-		if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
-			StatusActivityPubDeliver::dispatch($status);
-		}
-	}
+        if(config_cache('pixelfed.bouncer.enabled')) {
+            Bouncer::get($status);
+        }
+
+        Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
+        $hideNsfw = config('instance.hide_nsfw_on_public_feeds');
+        if( $status->uri == null &&
+            $status->scope == 'public' &&
+            in_array($status->type, $types) &&
+            $status->in_reply_to_id === null &&
+            $status->reblog_of_id === null &&
+            ($hideNsfw ? $status->is_nsfw == false : true)
+        ) {
+            if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
+                PublicTimelineService::add($status->id);
+            }
+        }
+
+        if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
+            StatusActivityPubDeliver::dispatch($status);
+        }
+    }
 }

app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php

@@ -90,7 +90,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
 			]);
 
 		$nm->each(function($n, $key) use($status) {
-			$res = Http::retry(3, 100, throw: false)->head($n['url']);
+			$res = Http::withOptions(['allow_redirects' => false])->retry(3, 100, throw: false)->head($n['url']);
 
 			if(!$res->successful()) {
 				return;

app/Jobs/StatusPipeline/StatusTagsPipeline.php

@@ -20,113 +20,119 @@ use App\Util\ActivityPub\Helpers;
 
 class StatusTagsPipeline implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-	protected $activity;
-	protected $status;
+    protected $activity;
+    protected $status;
 
-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct($activity, $status)
-	{
-		$this->activity = $activity;
-		$this->status = $status;
-	}
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($activity, $status)
+    {
+        $this->activity = $activity;
+        $this->status = $status;
+    }
 
-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$res = $this->activity;
-		$status = $this->status;
-		$tags = collect($res['tag']);
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $res = $this->activity;
+        $status = $this->status;
 
-		// Emoji
-		$tags->filter(function($tag) {
-			return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
-		})
-		->map(function($tag) {
-			CustomEmojiService::import($tag['id'], $this->status->id);
-		});
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
 
-		// Hashtags
-		$tags->filter(function($tag) {
-			return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
-		})
-		->map(function($tag) use($status) {
-			$name = substr($tag['name'], 0, 1) == '#' ?
-				substr($tag['name'], 1) : $tag['name'];
+        $tags = collect($res['tag']);
 
-			$banned = TrendingHashtagService::getBannedHashtagNames();
+        // Emoji
+        $tags->filter(function($tag) {
+            return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
+        })
+        ->map(function($tag) {
+            CustomEmojiService::import($tag['id'], $this->status->id);
+        });
 
-			if(count($banned)) {
+        // Hashtags
+        $tags->filter(function($tag) {
+            return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
+        })
+        ->map(function($tag) use($status) {
+            $name = substr($tag['name'], 0, 1) == '#' ?
+                substr($tag['name'], 1) : $tag['name'];
+
+            $banned = TrendingHashtagService::getBannedHashtagNames();
+
+            if(count($banned)) {
                 if(in_array(strtolower($name), array_map('strtolower', $banned))) {
-                   	return;
+                    return;
                 }
             }
 
             if(config('database.default') === 'pgsql') {
-            	$hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name))
-            		->first();
+                $hashtag = Hashtag::where('name', 'ilike', $name)
+                    ->orWhere('slug', 'ilike', str_slug($name, '-', false))
+                    ->first();
 
-            	if(!$hashtag) {
-            		$hashtag = new Hashtag;
-            		$hashtag->name = $name;
-            		$hashtag->slug = str_slug($name);
-            		$hashtag->save();
-            	}
+                if(!$hashtag) {
+                    $hashtag = Hashtag::updateOrCreate([
+                        'slug' => str_slug($name, '-', false),
+                        'name' => $name
+                    ]);
+                }
             } else {
-				$hashtag = Hashtag::firstOrCreate([
-					'slug' => str_slug($name)
-				], [
-					'name' => $name
-				]);
+                $hashtag = Hashtag::updateOrCreate([
+                    'slug' => str_slug($name, '-', false),
+                    'name' => $name
+                ]);
             }
 
-			StatusHashtag::firstOrCreate([
-				'status_id' => $status->id,
-				'hashtag_id' => $hashtag->id,
-				'profile_id' => $status->profile_id,
-				'status_visibility' => $status->scope
-			]);
-		});
+            StatusHashtag::firstOrCreate([
+                'status_id' => $status->id,
+                'hashtag_id' => $hashtag->id,
+                'profile_id' => $status->profile_id,
+                'status_visibility' => $status->scope
+            ]);
+        });
 
-		// Mentions
-		$tags->filter(function($tag) {
-			return $tag &&
-				$tag['type'] == 'Mention' &&
-				isset($tag['href']) &&
-				substr($tag['href'], 0, 8) === 'https://';
-		})
-		->map(function($tag) use($status) {
-			if(Helpers::validateLocalUrl($tag['href'])) {
-				$parts = explode('/', $tag['href']);
-				if(!$parts) {
-					return;
-				}
-				$pid = AccountService::usernameToId(end($parts));
-				if(!$pid) {
-					return;
-				}
-			} else {
-				$acct = Helpers::profileFetch($tag['href']);
-				if(!$acct) {
-					return;
-				}
-				$pid = $acct->id;
-			}
-			$mention = new Mention;
-			$mention->status_id = $status->id;
-			$mention->profile_id = $pid;
-			$mention->save();
-			MentionPipeline::dispatch($status, $mention);
-		});
-	}
+        // Mentions
+        $tags->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Mention' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        })
+        ->map(function($tag) use($status) {
+            if(Helpers::validateLocalUrl($tag['href'])) {
+                $parts = explode('/', $tag['href']);
+                if(!$parts) {
+                    return;
+                }
+                $pid = AccountService::usernameToId(end($parts));
+                if(!$pid) {
+                    return;
+                }
+            } else {
+                $acct = Helpers::profileFetch($tag['href']);
+                if(!$acct) {
+                    return;
+                }
+                $pid = $acct->id;
+            }
+            $mention = new Mention;
+            $mention->status_id = $status->id;
+            $mention->profile_id = $pid;
+            $mention->save();
+            MentionPipeline::dispatch($status, $mention);
+        });
+
+        StatusService::refresh($status->id);
+    }
 }

app/Jobs/VideoPipeline/VideoHlsPipeline.php

@@ -0,0 +1,109 @@
+<?php
+
+namespace App\Jobs\VideoPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use FFMpeg\Format\Video\X264;
+use FFMpeg;
+use Cache;
+use App\Services\MediaService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class VideoHlsPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $media;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-hls:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-hls:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($media)
+    {
+        $this->media = $media;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $depCheck = Cache::rememberForever('video-pipeline:hls:depcheck', function() {
+            $bin = config('laravel-ffmpeg.ffmpeg.binaries');
+            $output = shell_exec($bin . ' -version');
+            if($output && preg_match('/ffmpeg version ([^\s]+)/', $output, $matches)) {
+                $version = $matches[1];
+                return (version_compare($version, config('laravel-ffmpeg.min_hls_version')) >= 0) ? 'ok' : false;
+            } else {
+                return false;
+            }
+        });
+
+        if(!$depCheck || $depCheck !== 'ok') {
+            return;
+        }
+
+        $media = $this->media;
+
+        $bitrate = (new X264)->setKiloBitrate(config('media.hls.bitrate') ?? 1000);
+
+        $mp4 = $media->media_path;
+        $man = str_replace('.mp4', '.m3u8', $mp4);
+
+        FFMpeg::fromDisk('local')
+            ->open($mp4)
+            ->exportForHLS()
+            ->setSegmentLength(16)
+            ->setKeyFrameInterval(48)
+            ->addFormat($bitrate)
+            ->save($man);
+
+        $media->hls_path = $man;
+        $media->hls_transcoded_at = now();
+        $media->save();
+
+        MediaService::del($media->status_id);
+        usleep(50000);
+        StatusService::del($media->status_id);
+
+        return;
+    }
+}

app/Jobs/VideoPipeline/VideoThumbnail.php

@@ -16,13 +16,46 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Util\Media\Blurhash;
 use App\Services\MediaService;
 use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class VideoThumbnail implements ShouldQueue
+class VideoThumbnail implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     protected $media;
 
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-thumb:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-thumb:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
     /**
      * Create a new job instance.
      *
@@ -54,7 +87,7 @@ class VideoThumbnail implements ShouldQueue
             $path[$i] = $t;
             $save = implode('/', $path);
             $video = FFMpeg::open($base)
-            ->getFrameFromSeconds(0)
+            ->getFrameFromSeconds(1)
             ->export()
             ->toDisk('local')
             ->save($save);
@@ -68,6 +101,9 @@ class VideoThumbnail implements ShouldQueue
                 $media->save();
             }
 
+            if(config('media.hls.enabled')) {
+                VideoHlsPipeline::dispatch($media)->onQueue('mmo');
+            }
         } catch (Exception $e) {
             
         }

app/Models/AdminShadowFilter.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use App\Services\AccountService;
+use App\Profile;
+
+class AdminShadowFilter extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    protected $casts = [
+        'created_at' => 'datetime'
+    ];
+
+    public function account()
+    {
+        if($this->item_type === 'App\Profile') {
+            return AccountService::get($this->item_id, true);
+        }
+
+        return;
+    }
+
+    public function profile()
+    {
+        return $this->belongsTo(Profile::class, 'item_id');
+    }
+}

app/Models/DefaultDomainBlock.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class DefaultDomainBlock extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+}

app/Models/HashtagRelated.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HashtagRelated extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    /**
+     * The attributes that should be mutated to dates and other custom formats.
+     *
+     * @var array
+     */
+    protected $casts = [
+        'related_tags' => 'array',
+        'last_calculated_at' => 'datetime',
+        'last_moderated_at' => 'datetime',
+    ];
+}

app/Models/UserDomainBlock.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use App\Profile;
+
+class UserDomainBlock extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    public $timestamps = false;
+
+    public function profile()
+    {
+        return $this->belongsTo(Profile::class, 'profile_id');
+    }
+}

app/Observers/FollowerObserver.php

@@ -5,6 +5,8 @@ namespace App\Observers;
 use App\Follower;
 use App\Services\FollowerService;
 use Cache;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class FollowerObserver
 {
@@ -21,6 +23,7 @@ class FollowerObserver
         }
 
         FollowerService::add($follower->profile_id, $follower->following_id);
+        FeedFollowPipeline::dispatch($follower->profile_id, $follower->following_id)->onQueue('follow');
     }
 
     /**

app/Observers/HashtagFollowObserver.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Observers;
+
+use App\HashtagFollow;
+use App\Services\HashtagFollowService;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
+
+class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
+{
+    /**
+     * Handle the HashtagFollow "created" event.
+     */
+    public function created(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::add($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "updated" event.
+     */
+    public function updated(HashtagFollow $hashtagFollow): void
+    {
+    	//
+    }
+
+    /**
+     * Handle the HashtagFollow "deleting" event.
+     */
+    public function deleting(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "restored" event.
+     */
+    public function restored(HashtagFollow $hashtagFollow): void
+    {
+        //
+    }
+
+    /**
+     * Handle the HashtagFollow "force deleted" event.
+     */
+    public function forceDeleted(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+}

app/Observers/StatusHashtagObserver.php

@@ -5,32 +5,31 @@ namespace App\Observers;
 use DB;
 use App\StatusHashtag;
 use App\Services\StatusHashtagService;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagRemoveFanoutPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
 
-class StatusHashtagObserver
+class StatusHashtagObserver implements ShouldHandleEventsAfterCommit
 {
-    /**
-     * Handle events after all transactions are committed.
-     *
-     * @var bool
-     */
-    public $afterCommit = true;
-
     /**
      * Handle the notification "created" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function created(StatusHashtag $hashtag)
     {
         StatusHashtagService::set($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->increment('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagInsertFanoutPipeline::dispatch($hashtag)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "updated" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function updated(StatusHashtag $hashtag)
@@ -41,19 +40,22 @@ class StatusHashtagObserver
     /**
      * Handle the notification "deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function deleted(StatusHashtag $hashtag)
     {
         StatusHashtagService::del($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->decrement('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagRemoveFanoutPipeline::dispatch($hashtag->status_id, $hashtag->hashtag_id)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "restored" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function restored(StatusHashtag $hashtag)
@@ -64,7 +66,7 @@ class StatusHashtagObserver
     /**
      * Handle the notification "force deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function forceDeleted(StatusHashtag $hashtag)

app/Observers/StatusObserver.php

@@ -7,6 +7,8 @@ use App\Services\ProfileStatusService;
 use Cache;
 use App\Models\ImportPost;
 use App\Services\ImportService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class StatusObserver
 {
@@ -63,6 +65,14 @@ class StatusObserver
             ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete();
             ImportService::clearImportedFiles($status->profile_id);
         }
+
+        if(config('exp.cached_home_timeline')) {
+        	if($status->uri) {
+        		FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	} else {
+        		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	}
+        }
     }
 
     /**

app/Observers/UserFilterObserver.php

@@ -4,6 +4,8 @@ namespace App\Observers;
 
 use App\UserFilter;
 use App\Services\UserFilterService;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UserFilterObserver
 {
@@ -78,10 +80,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::mute($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::block($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}
@@ -96,10 +100,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::unmute($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::unblock($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}

app/Observers/UserObserver.php

@@ -7,90 +7,52 @@ use App\Follower;
 use App\Profile;
 use App\User;
 use App\UserSetting;
+use App\Services\UserFilterService;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
 use App\Jobs\FollowPipeline\FollowPipeline;
 use DB;
 use App\Services\FollowerService;
 
 class UserObserver
 {
-	/**
-	 * Listen to the User created event.
-	 *
-	 * @param \App\User $user
-	 *
-	 * @return void
-	 */
-	public function saved(User $user)
-	{
-		if($user->status == 'deleted') {
-			return;
-		}
+    /**
+     * Handle the notification "created" event.
+     *
+     * @param  \App\User $user
+     * @return void
+     */
+    public function created(User $user): void
+    {
+        $this->handleUser($user);
+    }
 
-        if(Profile::whereUsername($user->username)->exists()) {
-            return;
+    /**
+     * Listen to the User saved event.
+     *
+     * @param \App\User $user
+     *
+     * @return void
+     */
+    public function saved(User $user)
+    {
+        $this->handleUser($user);
+    }
+
+    /**
+     * Listen to the User updated event.
+     *
+     * @param \App\User $user
+     *
+     * @return void
+     */
+    public function updated(User $user): void
+    {
+        $this->handleUser($user);
+        if($user->profile) {
+            $this->applyDefaultDomainBlocks($user);
         }
-
-		if (empty($user->profile)) {
-			$profile = DB::transaction(function() use($user) {
-				$profile = new Profile();
-				$profile->user_id = $user->id;
-				$profile->username = $user->username;
-				$profile->name = $user->name;
-				$pkiConfig = [
-					'digest_alg'       => 'sha512',
-					'private_key_bits' => 2048,
-					'private_key_type' => OPENSSL_KEYTYPE_RSA,
-				];
-				$pki = openssl_pkey_new($pkiConfig);
-				openssl_pkey_export($pki, $pki_private);
-				$pki_public = openssl_pkey_get_details($pki);
-				$pki_public = $pki_public['key'];
-
-				$profile->private_key = $pki_private;
-				$profile->public_key = $pki_public;
-				$profile->save();
-				return $profile;
-			});
-
-			DB::transaction(function() use($user, $profile) {
-				$user = User::findOrFail($user->id);
-				$user->profile_id = $profile->id;
-				$user->save();
-
-				CreateAvatar::dispatch($profile);
-			});
-
-			if(config_cache('account.autofollow') == true) {
-				$names = config_cache('account.autofollow_usernames');
-				$names = explode(',', $names);
-
-				if(!$names || !last($names)) {
-					return;
-				}
-
-				$profiles = Profile::whereIn('username', $names)->get();
-
-				if($profiles) {
-					foreach($profiles as $p) {
-						$follower = new Follower;
-						$follower->profile_id = $profile->id;
-						$follower->following_id = $p->id;
-						$follower->save();
-
-						FollowPipeline::dispatch($follower);
-					}
-				}
-			}
-		}
-
-		if (empty($user->settings)) {
-			DB::transaction(function() use($user) {
-				UserSetting::firstOrCreate([
-					'user_id' => $user->id
-				]);
-			});
-		}
-	}
+    }
 
     /**
      * Handle the user "deleted" event.
@@ -102,4 +64,97 @@ class UserObserver
     {
         FollowerService::delCache($user->profile_id);
     }
+
+    protected function handleUser($user)
+    {
+        if(in_array($user->status, ['deleted', 'delete'])) {
+            return;
+        }
+
+        if(Profile::whereUsername($user->username)->exists()) {
+            return;
+        }
+
+        if (empty($user->profile)) {
+            $profile = DB::transaction(function() use($user) {
+                $profile = new Profile();
+                $profile->user_id = $user->id;
+                $profile->username = $user->username;
+                $profile->name = $user->name;
+                $pkiConfig = [
+                    'digest_alg'       => 'sha512',
+                    'private_key_bits' => 2048,
+                    'private_key_type' => OPENSSL_KEYTYPE_RSA,
+                ];
+                $pki = openssl_pkey_new($pkiConfig);
+                openssl_pkey_export($pki, $pki_private);
+                $pki_public = openssl_pkey_get_details($pki);
+                $pki_public = $pki_public['key'];
+
+                $profile->private_key = $pki_private;
+                $profile->public_key = $pki_public;
+                $profile->save();
+                $this->applyDefaultDomainBlocks($user);
+                return $profile;
+            });
+
+
+            DB::transaction(function() use($user, $profile) {
+                $user = User::findOrFail($user->id);
+                $user->profile_id = $profile->id;
+                $user->save();
+
+                CreateAvatar::dispatch($profile);
+            });
+
+            if(config_cache('account.autofollow') == true) {
+                $names = config_cache('account.autofollow_usernames');
+                $names = explode(',', $names);
+
+                if(!$names || !last($names)) {
+                    return;
+                }
+
+                $profiles = Profile::whereIn('username', $names)->get();
+
+                if($profiles) {
+                    foreach($profiles as $p) {
+                        $follower = new Follower;
+                        $follower->profile_id = $profile->id;
+                        $follower->following_id = $p->id;
+                        $follower->save();
+
+                        FollowPipeline::dispatch($follower);
+                    }
+                }
+            }
+        }
+
+        if (empty($user->settings)) {
+            DB::transaction(function() use($user) {
+                UserSetting::firstOrCreate([
+                    'user_id' => $user->id
+                ]);
+            });
+        }
+    }
+
+    protected function applyDefaultDomainBlocks($user)
+    {
+        if($user->profile_id == null) {
+            return;
+        }
+        $defaultDomainBlocks = DefaultDomainBlock::pluck('domain')->toArray();
+
+        if(!$defaultDomainBlocks || !count($defaultDomainBlocks)) {
+            return;
+        }
+
+        foreach($defaultDomainBlocks as $domain) {
+            UserDomainBlock::updateOrCreate([
+                'profile_id' => $user->profile_id,
+                'domain' => strtolower(trim($domain))
+            ]);
+        }
+    }
 }

app/Services/AccountService.php

@@ -7,6 +7,7 @@ use App\Profile;
 use App\Status;
 use App\User;
 use App\UserSetting;
+use App\Models\UserDomainBlock;
 use App\Transformer\Api\AccountTransformer;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
@@ -15,209 +16,232 @@ use Illuminate\Support\Str;
 
 class AccountService
 {
-	const CACHE_KEY = 'pf:services:account:';
+    const CACHE_KEY = 'pf:services:account:';
 
-	public static function get($id, $softFail = false)
-	{
-		$res = Cache::remember(self::CACHE_KEY . $id, 43200, function() use($id) {
-			$fractal = new Fractal\Manager();
-			$fractal->setSerializer(new ArraySerializer());
-			$profile = Profile::find($id);
-			if(!$profile || $profile->status === 'delete') {
-				return null;
-			}
-			$resource = new Fractal\Resource\Item($profile, new AccountTransformer());
-			return $fractal->createData($resource)->toArray();
-		});
+    public static function get($id, $softFail = false)
+    {
+        $res = Cache::remember(self::CACHE_KEY . $id, 43200, function() use($id) {
+            $fractal = new Fractal\Manager();
+            $fractal->setSerializer(new ArraySerializer());
+            $profile = Profile::find($id);
+            if(!$profile || $profile->status === 'delete') {
+                return null;
+            }
+            $resource = new Fractal\Resource\Item($profile, new AccountTransformer());
+            return $fractal->createData($resource)->toArray();
+        });
 
-		if(!$res) {
-			return $softFail ? null : abort(404);
-		}
-		return $res;
-	}
+        if(!$res) {
+            return $softFail ? null : abort(404);
+        }
+        return $res;
+    }
 
-	public static function getMastodon($id, $softFail = false)
-	{
-		$account = self::get($id, $softFail);
-		if(!$account) {
-			return null;
-		}
+    public static function getMastodon($id, $softFail = false)
+    {
+        $account = self::get($id, $softFail);
+        if(!$account) {
+            return null;
+        }
 
-		if(config('exp.emc') == false) {
-			return $account;
-		}
+        if(config('exp.emc') == false) {
+            return $account;
+        }
 
-		unset(
-			$account['header_bg'],
-			$account['is_admin'],
-			$account['last_fetched_at'],
-			$account['local'],
-			$account['location'],
-			$account['note_text'],
-			$account['pronouns'],
-			$account['website']
-		);
+        unset(
+            $account['header_bg'],
+            $account['is_admin'],
+            $account['last_fetched_at'],
+            $account['local'],
+            $account['location'],
+            $account['note_text'],
+            $account['pronouns'],
+            $account['website']
+        );
 
-		$account['avatar_static'] = $account['avatar'];
-		$account['bot'] = false;
-		$account['emojis'] = [];
-		$account['fields'] = [];
-		$account['header'] = url('/storage/headers/missing.png');
-		$account['header_static'] = url('/storage/headers/missing.png');
-		$account['last_status_at'] = null;
+        $account['avatar_static'] = $account['avatar'];
+        $account['bot'] = false;
+        $account['emojis'] = [];
+        $account['fields'] = [];
+        $account['header'] = url('/storage/headers/missing.png');
+        $account['header_static'] = url('/storage/headers/missing.png');
+        $account['last_status_at'] = null;
 
-		return $account;
-	}
+        return $account;
+    }
 
-	public static function del($id)
-	{
-		Cache::forget('pf:activitypub:user-object:by-id:' . $id);
-		return Cache::forget(self::CACHE_KEY . $id);
-	}
+    public static function del($id)
+    {
+        Cache::forget('pf:activitypub:user-object:by-id:' . $id);
+        return Cache::forget(self::CACHE_KEY . $id);
+    }
 
-	public static function settings($id)
-	{
-		return Cache::remember('profile:compose:settings:' . $id, 604800, function() use($id) {
-			$settings = UserSetting::whereUserId($id)->first();
-			if(!$settings) {
-				return self::defaultSettings();
-			}
-			return collect($settings)
-			->filter(function($item, $key) {
-				return in_array($key, array_keys(self::defaultSettings())) == true;
-			})
-			->map(function($item, $key) {
-				if($key == 'compose_settings') {
-					$cs = self::defaultSettings()['compose_settings'];
-					$ms = is_array($item) ? $item : [];
-					return array_merge($cs, $ms);
-				}
+    public static function settings($id)
+    {
+        return Cache::remember('profile:compose:settings:' . $id, 604800, function() use($id) {
+            $settings = UserSetting::whereUserId($id)->first();
+            if(!$settings) {
+                return self::defaultSettings();
+            }
+            return collect($settings)
+            ->filter(function($item, $key) {
+                return in_array($key, array_keys(self::defaultSettings())) == true;
+            })
+            ->map(function($item, $key) {
+                if($key == 'compose_settings') {
+                    $cs = self::defaultSettings()['compose_settings'];
+                    $ms = is_array($item) ? $item : [];
+                    return array_merge($cs, $ms);
+                }
 
-				if($key == 'other') {
-					$other =  self::defaultSettings()['other'];
-					$mo = is_array($item) ? $item : [];
-					return array_merge($other, $mo);
-				}
-				return $item;
-			});
-		});
-	}
+                if($key == 'other') {
+                    $other =  self::defaultSettings()['other'];
+                    $mo = is_array($item) ? $item : [];
+                    return array_merge($other, $mo);
+                }
+                return $item;
+            });
+        });
+    }
 
-	public static function canEmbed($id)
-	{
-		return self::settings($id)['other']['disable_embeds'] == false;
-	}
+    public static function canEmbed($id)
+    {
+        return self::settings($id)['other']['disable_embeds'] == false;
+    }
 
-	public static function defaultSettings()
-	{
-		return [
-			'crawlable' => true,
-			'public_dm' => false,
-			'reduce_motion' => false,
-			'high_contrast_mode' => false,
-			'video_autoplay' => false,
-			'show_profile_follower_count' => true,
-			'show_profile_following_count' => true,
-			'compose_settings' => [
-				'default_scope' => 'public',
-				'default_license' => 1,
-				'media_descriptions' => false
-			],
-			'other' => [
-				'advanced_atom' => false,
-				'disable_embeds' => false,
-				'mutual_mention_notifications' => false,
-				'hide_collections' => false,
-				'hide_like_counts' => false,
-				'hide_groups' => false,
-				'hide_stories' => false,
-				'disable_cw' => false,
-			]
-		];
-	}
+    public static function defaultSettings()
+    {
+        return [
+            'crawlable' => true,
+            'public_dm' => false,
+            'reduce_motion' => false,
+            'high_contrast_mode' => false,
+            'video_autoplay' => false,
+            'show_profile_follower_count' => true,
+            'show_profile_following_count' => true,
+            'compose_settings' => [
+                'default_scope' => 'public',
+                'default_license' => 1,
+                'media_descriptions' => false
+            ],
+            'other' => [
+                'advanced_atom' => false,
+                'disable_embeds' => false,
+                'mutual_mention_notifications' => false,
+                'hide_collections' => false,
+                'hide_like_counts' => false,
+                'hide_groups' => false,
+                'hide_stories' => false,
+                'disable_cw' => false,
+            ]
+        ];
+    }
 
-	public static function syncPostCount($id)
-	{
-		$profile = Profile::find($id);
+    public static function syncPostCount($id)
+    {
+        $profile = Profile::find($id);
 
-		if(!$profile) {
-			return false;
-		}
+        if(!$profile) {
+            return false;
+        }
 
-		$key = self::CACHE_KEY . 'pcs:' . $id;
+        $key = self::CACHE_KEY . 'pcs:' . $id;
 
-		if(Cache::has($key)) {
-			return;
-		}
+        if(Cache::has($key)) {
+            return;
+        }
 
-		$count = Status::whereProfileId($id)
-			->whereNull('in_reply_to_id')
-			->whereNull('reblog_of_id')
-			->whereIn('scope', ['public', 'unlisted', 'private'])
-			->count();
+        $count = Status::whereProfileId($id)
+            ->whereNull('in_reply_to_id')
+            ->whereNull('reblog_of_id')
+            ->whereIn('scope', ['public', 'unlisted', 'private'])
+            ->count();
 
-		$profile->status_count = $count;
-		$profile->save();
+        $profile->status_count = $count;
+        $profile->save();
 
-		Cache::put($key, 1, 900);
-		return true;
-	}
+        Cache::put($key, 1, 900);
+        return true;
+    }
 
-	public static function usernameToId($username)
-	{
-		$key = self::CACHE_KEY . 'u2id:' . hash('sha256', $username);
-		return Cache::remember($key, 900, function() use($username) {
-			$s = Str::of($username);
-			if($s->contains('@') && !$s->startsWith('@')) {
-				$username = "@{$username}";
-			}
-			$profile = DB::table('profiles')
-				->whereUsername($username)
-				->first();
-			if(!$profile) {
-				return null;
-			}
-			return (string) $profile->id;
-		});
-	}
+    public static function usernameToId($username)
+    {
+        $key = self::CACHE_KEY . 'u2id:' . hash('sha256', $username);
+        return Cache::remember($key, 14400, function() use($username) {
+            $s = Str::of($username);
+            if($s->contains('@') && !$s->startsWith('@')) {
+                $username = "@{$username}";
+            }
+            $profile = DB::table('profiles')
+                ->whereUsername($username)
+                ->first();
+            if(!$profile) {
+                return null;
+            }
+            return (string) $profile->id;
+        });
+    }
 
-	public static function hiddenFollowers($id)
-	{
-		$account = self::get($id, true);
-		if(!$account || !isset($account['local']) || $account['local'] == false) {
-			return false;
-		}
+    public static function hiddenFollowers($id)
+    {
+        $account = self::get($id, true);
+        if(!$account || !isset($account['local']) || $account['local'] == false) {
+            return false;
+        }
 
-		return Cache::remember('pf:acct:settings:hidden-followers:' . $id, 43200, function() use($id) {
-			$user = User::whereProfileId($id)->first();
-			if(!$user) {
-				return false;
-			}
-			$settings = UserSetting::whereUserId($user->id)->first();
-			if($settings) {
-				return $settings->show_profile_follower_count == false;
-			}
-			return false;
-		});
-	}
+        return Cache::remember('pf:acct:settings:hidden-followers:' . $id, 43200, function() use($id) {
+            $user = User::whereProfileId($id)->first();
+            if(!$user) {
+                return false;
+            }
+            $settings = UserSetting::whereUserId($user->id)->first();
+            if($settings) {
+                return $settings->show_profile_follower_count == false;
+            }
+            return false;
+        });
+    }
 
-	public static function hiddenFollowing($id)
-	{
-		$account = self::get($id, true);
-		if(!$account || !isset($account['local']) || $account['local'] == false) {
-			return false;
-		}
+    public static function hiddenFollowing($id)
+    {
+        $account = self::get($id, true);
+        if(!$account || !isset($account['local']) || $account['local'] == false) {
+            return false;
+        }
 
-		return Cache::remember('pf:acct:settings:hidden-following:' . $id, 43200, function() use($id) {
-			$user = User::whereProfileId($id)->first();
-			if(!$user) {
-				return false;
-			}
-			$settings = UserSetting::whereUserId($user->id)->first();
-			if($settings) {
-				return $settings->show_profile_following_count == false;
-			}
-			return false;
-		});
-	}
+        return Cache::remember('pf:acct:settings:hidden-following:' . $id, 43200, function() use($id) {
+            $user = User::whereProfileId($id)->first();
+            if(!$user) {
+                return false;
+            }
+            $settings = UserSetting::whereUserId($user->id)->first();
+            if($settings) {
+                return $settings->show_profile_following_count == false;
+            }
+            return false;
+        });
+    }
+
+    public static function setLastActive($id = false)
+    {
+        if(!$id) { return; }
+        $key = 'user:last_active_at:id:' . $id;
+        if(!Cache::has($key)) {
+            $user = User::find($id);
+            if(!$user) { return; }
+            $user->last_active_at = now();
+            $user->save();
+            Cache::put($key, 1, 14400);
+        }
+        return;
+    }
+
+    public static function blocksDomain($pid, $domain = false)
+    {
+        if(!$domain) {
+            return;
+        }
+
+        return UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->exists();
+    }
 }

app/Services/ActivityPubFetchService.php

@@ -28,7 +28,7 @@ class ActivityPubFetchService
 		$headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
 
 		try {
-			$res = Http::withHeaders($headers)
+			$res = Http::withOptions(['allow_redirects' => false])->withHeaders($headers)
 				->timeout(30)
 				->connectTimeout(5)
 				->retry(3, 500)

app/Services/AdminShadowFilterService.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\AdminShadowFilter;
+use Cache;
+
+class AdminShadowFilterService
+{
+    const CACHE_KEY = 'pf:services:asfs:';
+
+    public static function queryFilter($name = 'hide_from_public_feeds')
+    {
+        return AdminShadowFilter::whereItemType('App\Profile')
+            ->whereActive(1)
+            ->where('hide_from_public_feeds', true)
+            ->pluck('item_id')
+            ->toArray();
+    }
+
+    public static function getHideFromPublicFeedsList($refresh = false)
+    {
+        $key = self::CACHE_KEY . 'list:hide_from_public_feeds';
+        if($refresh) {
+            Cache::forget($key);
+        }
+        return Cache::remember($key, 86400, function() {
+            return AdminShadowFilter::whereItemType('App\Profile')
+                ->whereActive(1)
+                ->where('hide_from_public_feeds', true)
+                ->pluck('item_id')
+                ->toArray();
+        });
+    }
+
+    public static function canAddToPublicFeedByProfileId($profileId)
+    {
+        return !in_array($profileId, self::getHideFromPublicFeedsList());
+    }
+
+    public static function refresh()
+    {
+        $keys = [
+            self::CACHE_KEY . 'list:hide_from_public_feeds'
+        ];
+
+        foreach($keys as $key) {
+            Cache::forget($key);
+        }
+    }
+}

app/Services/AvatarService.php

@@ -3,21 +3,125 @@
 namespace App\Services;
 
 use Cache;
+use Storage;
+use Illuminate\Support\Str;
+use App\Avatar;
 use App\Profile;
+use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
+use League\Flysystem\UnableToCheckDirectoryExistence;
+use League\Flysystem\UnableToRetrieveMetadata;
 
 class AvatarService
 {
-	public static function get($profile_id)
-	{
-		$exists = Cache::get('avatar:' . $profile_id);
-		if($exists) {
-			return $exists;
-		}
+    public static function get($profile_id)
+    {
+        $exists = Cache::get('avatar:' . $profile_id);
+        if($exists) {
+            return $exists;
+        }
 
-		$profile = Profile::find($profile_id);
-		if(!$profile) {
-			return config('app.url') . '/storage/avatars/default.jpg';
-		}
-		return $profile->avatarUrl();
-	}
+        $profile = Profile::find($profile_id);
+        if(!$profile) {
+            return config('app.url') . '/storage/avatars/default.jpg';
+        }
+        return $profile->avatarUrl();
+    }
+
+    public static function disk()
+    {
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return false;
+        }
+
+        $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
+        $disk = Storage::disk($driver);
+
+        return $disk;
+    }
+
+    public static function storage(Avatar $avatar)
+    {
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        return $disk->allFiles($base . $avatar->profile_id);
+    }
+
+    public static function cleanup($avatar, $confirm = false)
+    {
+        if(!$avatar || !$confirm) {
+            return;
+        }
+
+        if($avatar->cdn_url == null) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return;
+        }
+
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        try {
+            $exists = $disk->directoryExists($base . $avatar->profile_id);
+        } catch (
+            UnableToRetrieveMetadata |
+            UnableToCheckDirectoryExistence |
+            Exception $e
+        ) {
+            return;
+        }
+
+        if(!$exists) {
+            return;
+        }
+
+        $files = collect($disk->allFiles($base . $avatar->profile_id));
+
+        if(!$files || !$files->count() || $files->count() === 1) {
+            return;
+        }
+
+        if($files->count() > 5) {
+            AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
+            return;
+        }
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
 }

app/Services/FollowerService.php

@@ -6,206 +6,245 @@ use Illuminate\Support\Facades\Redis;
 use Cache;
 use DB;
 use App\{
-	Follower,
-	Profile,
-	User
+    Follower,
+    Profile,
+    User
 };
 use App\Jobs\FollowPipeline\FollowServiceWarmCache;
 
 class FollowerService
 {
-	const CACHE_KEY = 'pf:services:followers:';
-	const FOLLOWERS_SYNC_KEY = 'pf:services:followers:sync-followers:';
-	const FOLLOWING_SYNC_KEY = 'pf:services:followers:sync-following:';
-	const FOLLOWING_KEY = 'pf:services:follow:following:id:';
-	const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
+    const CACHE_KEY = 'pf:services:followers:';
+    const FOLLOWERS_SYNC_KEY = 'pf:services:followers:sync-followers:';
+    const FOLLOWING_SYNC_KEY = 'pf:services:followers:sync-following:';
+    const FOLLOWING_KEY = 'pf:services:follow:following:id:';
+    const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
+    const FOLLOWERS_LOCAL_KEY = 'pf:services:follow:local-follower-ids:';
+    const FOLLOWERS_INTER_KEY = 'pf:services:follow:followers:inter:id:';
 
-	public static function add($actor, $target)
-	{
-		$ts = (int) microtime(true);
-		RelationshipService::refresh($actor, $target);
-		Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target);
-		Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor);
-		Cache::forget('profile:following:' . $actor);
-	}
+    public static function add($actor, $target, $refresh = true)
+    {
+        $ts = (int) microtime(true);
+        if($refresh) {
+          RelationshipService::refresh($actor, $target);
+        } else {
+          RelationshipService::forget($actor, $target);
+        }
+        Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target);
+        Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor);
+        Cache::forget('profile:following:' . $actor);
+    }
 
-	public static function remove($actor, $target)
-	{
-		Redis::zrem(self::FOLLOWING_KEY . $actor, $target);
-		Redis::zrem(self::FOLLOWERS_KEY . $target, $actor);
-		Cache::forget('pf:services:follower:audience:' . $actor);
-		Cache::forget('pf:services:follower:audience:' . $target);
-		AccountService::del($actor);
-		AccountService::del($target);
-		RelationshipService::refresh($actor, $target);
-		Cache::forget('profile:following:' . $actor);
-	}
+    public static function remove($actor, $target, $silent = false)
+    {
+        Redis::zrem(self::FOLLOWING_KEY . $actor, $target);
+        Redis::zrem(self::FOLLOWERS_KEY . $target, $actor);
+        if($silent !== true) {
+            AccountService::del($actor);
+            AccountService::del($target);
+            RelationshipService::refresh($actor, $target);
+            Cache::forget('profile:following:' . $actor);
+        } else {
+            RelationshipService::forget($actor, $target);
+        }
+    }
 
-	public static function followers($id, $start = 0, $stop = 10)
-	{
-		self::cacheSyncCheck($id, 'followers');
-		return Redis::zrevrange(self::FOLLOWERS_KEY . $id, $start, $stop);
-	}
+    public static function followers($id, $start = 0, $stop = 10)
+    {
+        self::cacheSyncCheck($id, 'followers');
+        return Redis::zrevrange(self::FOLLOWERS_KEY . $id, $start, $stop);
+    }
 
-	public static function following($id, $start = 0, $stop = 10)
-	{
-		self::cacheSyncCheck($id, 'following');
-		return Redis::zrevrange(self::FOLLOWING_KEY . $id, $start, $stop);
-	}
+    public static function following($id, $start = 0, $stop = 10)
+    {
+        self::cacheSyncCheck($id, 'following');
+        return Redis::zrevrange(self::FOLLOWING_KEY . $id, $start, $stop);
+    }
 
-	public static function followersPaginate($id, $page = 1, $limit = 10)
-	{
-		$start = $page == 1 ? 0 : $page * $limit - $limit;
-		$end = $start + ($limit - 1);
-		return self::followers($id, $start, $end);
-	}
+    public static function followersPaginate($id, $page = 1, $limit = 10)
+    {
+        $start = $page == 1 ? 0 : $page * $limit - $limit;
+        $end = $start + ($limit - 1);
+        return self::followers($id, $start, $end);
+    }
 
-	public static function followingPaginate($id, $page = 1, $limit = 10)
-	{
-		$start = $page == 1 ? 0 : $page * $limit - $limit;
-		$end = $start + ($limit - 1);
-		return self::following($id, $start, $end);
-	}
+    public static function followingPaginate($id, $page = 1, $limit = 10)
+    {
+        $start = $page == 1 ? 0 : $page * $limit - $limit;
+        $end = $start + ($limit - 1);
+        return self::following($id, $start, $end);
+    }
 
-	public static function followerCount($id, $warmCache = true)
-	{
-		if($warmCache) {
-			self::cacheSyncCheck($id, 'followers');
-		}
-		return Redis::zCard(self::FOLLOWERS_KEY . $id);
-	}
+    public static function followerCount($id, $warmCache = true)
+    {
+        if($warmCache) {
+            self::cacheSyncCheck($id, 'followers');
+        }
+        return Redis::zCard(self::FOLLOWERS_KEY . $id);
+    }
 
-	public static function followingCount($id, $warmCache = true)
-	{
-		if($warmCache) {
-			self::cacheSyncCheck($id, 'following');
-		}
-		return Redis::zCard(self::FOLLOWING_KEY . $id);
-	}
+    public static function followingCount($id, $warmCache = true)
+    {
+        if($warmCache) {
+            self::cacheSyncCheck($id, 'following');
+        }
+        return Redis::zCard(self::FOLLOWING_KEY . $id);
+    }
 
-	public static function follows(string $actor, string $target)
-	{
-		if($actor == $target) {
-			return false;
-		}
+    public static function follows(string $actor, string $target, $quickCheck = false)
+    {
+        if($actor == $target) {
+            return false;
+        }
 
-		if(self::followerCount($target, false) && self::followingCount($actor, false)) {
-			self::cacheSyncCheck($target, 'followers');
-			return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
-		} else {
-			self::cacheSyncCheck($target, 'followers');
-			self::cacheSyncCheck($actor, 'following');
-			return Follower::whereProfileId($actor)->whereFollowingId($target)->exists();
-		}
-	}
+        if($quickCheck) {
+            return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
+        }
 
-	public static function cacheSyncCheck($id, $scope = 'followers')
-	{
-		if($scope === 'followers') {
-			if(Cache::get(self::FOLLOWERS_SYNC_KEY . $id) != null) {
-				return;
-			}
-			FollowServiceWarmCache::dispatch($id)->onQueue('low');
-		}
-		if($scope === 'following') {
-			if(Cache::get(self::FOLLOWING_SYNC_KEY . $id) != null) {
-				return;
-			}
-			FollowServiceWarmCache::dispatch($id)->onQueue('low');
-		}
-		return;
-	}
+        if(self::followerCount($target, false) && self::followingCount($actor, false)) {
+            self::cacheSyncCheck($target, 'followers');
+            return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
+        } else {
+            self::cacheSyncCheck($target, 'followers');
+            self::cacheSyncCheck($actor, 'following');
+            return Follower::whereProfileId($actor)->whereFollowingId($target)->exists();
+        }
+    }
 
-	public static function audience($profile, $scope = null)
-	{
-		return (new self)->getAudienceInboxes($profile, $scope);
-	}
+    public static function cacheSyncCheck($id, $scope = 'followers')
+    {
+        if($scope === 'followers') {
+            if(Cache::get(self::FOLLOWERS_SYNC_KEY . $id) != null) {
+                return;
+            }
+            FollowServiceWarmCache::dispatch($id)->onQueue('low');
+        }
+        if($scope === 'following') {
+            if(Cache::get(self::FOLLOWING_SYNC_KEY . $id) != null) {
+                return;
+            }
+            FollowServiceWarmCache::dispatch($id)->onQueue('low');
+        }
+        return;
+    }
 
-	public static function softwareAudience($profile, $software = 'pixelfed')
-	{
-		return collect(self::audience($profile))
-			->filter(function($inbox) use($software) {
-				$domain = parse_url($inbox, PHP_URL_HOST);
-				if(!$domain) {
-					return false;
-				}
-				return InstanceService::software($domain) === strtolower($software);
-			})
-			->unique()
-			->values()
-			->toArray();
-	}
+    public static function audience($profile, $scope = null)
+    {
+        return (new self)->getAudienceInboxes($profile, $scope);
+    }
 
-	protected function getAudienceInboxes($pid, $scope = null)
-	{
-		$key = 'pf:services:follower:audience:' . $pid;
-		$domains = Cache::remember($key, 432000, function() use($pid) {
-			$profile = Profile::whereNull(['status', 'domain'])->find($pid);
-			if(!$profile) {
-				return [];
-			}
-			return $profile
-				->followers()
-				->get()
-				->map(function($follow) {
-					return $follow->sharedInbox ?? $follow->inbox_url;
-				})
-				->filter()
-				->unique()
-				->values();
-		});
+    public static function softwareAudience($profile, $software = 'pixelfed')
+    {
+        return collect(self::audience($profile))
+            ->filter(function($inbox) use($software) {
+                $domain = parse_url($inbox, PHP_URL_HOST);
+                if(!$domain) {
+                    return false;
+                }
+                return InstanceService::software($domain) === strtolower($software);
+            })
+            ->unique()
+            ->values()
+            ->toArray();
+    }
 
-		if(!$domains || !$domains->count()) {
-			return [];
-		}
+    protected function getAudienceInboxes($pid, $scope = null)
+    {
+        $key = 'pf:services:follower:audience:' . $pid;
+        $domains = Cache::remember($key, 432000, function() use($pid) {
+            $profile = Profile::whereNull(['status', 'domain'])->find($pid);
+            if(!$profile) {
+                return [];
+            }
+            return $profile
+                ->followers()
+                ->get()
+                ->map(function($follow) {
+                    return $follow->sharedInbox ?? $follow->inbox_url;
+                })
+                ->filter()
+                ->unique()
+                ->values();
+        });
 
-		$banned = InstanceService::getBannedDomains();
+        if(!$domains || !$domains->count()) {
+            return [];
+        }
 
-		if(!$banned || count($banned) === 0) {
-			return $domains->toArray();
-		}
+        $banned = InstanceService::getBannedDomains();
 
-		$res = $domains->filter(function($domain) use($banned) {
-			$parsed = parse_url($domain, PHP_URL_HOST);
-			return !in_array($parsed, $banned);
-		})
-		->values()
-		->toArray();
+        if(!$banned || count($banned) === 0) {
+            return $domains->toArray();
+        }
 
-		return $res;
-	}
+        $res = $domains->filter(function($domain) use($banned) {
+            $parsed = parse_url($domain, PHP_URL_HOST);
+            return !in_array($parsed, $banned);
+        })
+        ->values()
+        ->toArray();
 
-	public static function mutualCount($pid, $mid)
-	{
-		return Cache::remember(self::CACHE_KEY . ':mutualcount:' . $pid . ':' . $mid, 3600, function() use($pid, $mid) {
-			return DB::table('followers as u')
-				->join('followers as s', 'u.following_id', '=', 's.following_id')
-				->where('s.profile_id', $mid)
-				->where('u.profile_id', $pid)
-				->count();
-		});
-	}
+        return $res;
+    }
 
-	public static function mutualIds($pid, $mid, $limit = 3)
-	{
-		$key = self::CACHE_KEY . ':mutualids:' . $pid . ':' . $mid . ':limit_' . $limit;
-		return Cache::remember($key, 3600, function() use($pid, $mid, $limit) {
-			return DB::table('followers as u')
-				->join('followers as s', 'u.following_id', '=', 's.following_id')
-				->where('s.profile_id', $mid)
-				->where('u.profile_id', $pid)
-				->limit($limit)
-				->pluck('s.following_id')
-				->toArray();
-		});
-	}
+    public static function mutualCount($pid, $mid)
+    {
+        return Cache::remember(self::CACHE_KEY . ':mutualcount:' . $pid . ':' . $mid, 3600, function() use($pid, $mid) {
+            return DB::table('followers as u')
+                ->join('followers as s', 'u.following_id', '=', 's.following_id')
+                ->where('s.profile_id', $mid)
+                ->where('u.profile_id', $pid)
+                ->count();
+        });
+    }
 
-	public static function delCache($id)
-	{
-		Redis::del(self::CACHE_KEY . $id);
-		Redis::del(self::FOLLOWING_KEY . $id);
-		Redis::del(self::FOLLOWERS_KEY . $id);
-		Cache::forget(self::FOLLOWERS_SYNC_KEY . $id);
-		Cache::forget(self::FOLLOWING_SYNC_KEY . $id);
-	}
+    public static function mutualIds($pid, $mid, $limit = 3)
+    {
+        $key = self::CACHE_KEY . ':mutualids:' . $pid . ':' . $mid . ':limit_' . $limit;
+        return Cache::remember($key, 3600, function() use($pid, $mid, $limit) {
+            return DB::table('followers as u')
+                ->join('followers as s', 'u.following_id', '=', 's.following_id')
+                ->where('s.profile_id', $mid)
+                ->where('u.profile_id', $pid)
+                ->limit($limit)
+                ->pluck('s.following_id')
+                ->toArray();
+        });
+    }
+
+    public static function mutualAccounts($actorId, $profileId)
+    {
+        if($actorId == $profileId) {
+            return [];
+        }
+        $actorKey = self::FOLLOWING_KEY . $actorId;
+        $profileKey = self::FOLLOWERS_KEY . $profileId;
+        $key = self::FOLLOWERS_INTER_KEY . $actorId . ':' . $profileId;
+        $res = Redis::zinterstore($key, [$actorKey, $profileKey]);
+        if($res) {
+            return Redis::zrange($key, 0, -1);
+        } else {
+            return [];
+        }
+    }
+
+    public static function delCache($id)
+    {
+        Redis::del(self::CACHE_KEY . $id);
+        Redis::del(self::FOLLOWING_KEY . $id);
+        Redis::del(self::FOLLOWERS_KEY . $id);
+        Cache::forget(self::FOLLOWERS_SYNC_KEY . $id);
+        Cache::forget(self::FOLLOWING_SYNC_KEY . $id);
+    }
+
+    public static function localFollowerIds($pid, $limit = 0)
+    {
+        $key = self::FOLLOWERS_LOCAL_KEY . $pid;
+        $res = Cache::remember($key, 7200, function() use($pid) {
+            return DB::table('followers')->whereFollowingId($pid)->whereLocalProfile(true)->pluck('profile_id')->sort();
+        });
+        return $limit ?
+            $res->take($limit)->values()->toArray() :
+            $res->values()->toArray();
+    }
 }

app/Services/HashtagFollowService.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\HashtagFollow;
+
+class HashtagFollowService
+{
+	const FOLLOW_KEY = 'pf:services:hashtag-follows:v1:';
+	const CACHE_KEY = 'pf:services:hfs:byHid:';
+	const CACHE_WARMED = 'pf:services:hfs:wc:byHid';
+
+	public static function getPidByHid($hid)
+	{
+		if(!self::isWarm($hid)) {
+			return self::warmCache($hid);
+		}
+		return self::get($hid);
+	}
+
+	public static function unfollow($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function add($hid, $pid)
+	{
+		return Redis::zadd(self::CACHE_KEY . $hid, $pid, $pid);
+	}
+
+	public static function rem($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function get($hid)
+	{
+		return Redis::zrange(self::CACHE_KEY . $hid, 0, -1);
+	}
+
+	public static function count($hid)
+	{
+		return Redis::zcard(self::CACHE_KEY . $hid);
+	}
+
+	public static function warmCache($hid)
+	{
+		foreach(HashtagFollow::whereHashtagId($hid)->lazyById(20, 'id') as $h) {
+			if($h) {
+				self::add($h->hashtag_id, $h->profile_id);
+			}
+		}
+
+		self::setWarm($hid);
+
+		return self::get($hid);
+	}
+
+	public static function isWarm($hid)
+	{
+		return Redis::zcount(self::CACHE_KEY . $hid, 0, -1) ?? Redis::zscore(self::CACHE_WARMED, $hid) != null;
+	}
+
+	public static function setWarm($hid)
+	{
+		return Redis::zadd(self::CACHE_WARMED, $hid, $hid);
+	}
+}

app/Services/HashtagRelatedService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use DB;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+
+class HashtagRelatedService
+{
+    public static function get($id)
+    {
+        $tag = HashtagRelated::whereHashtagId($id)->first();
+        if(!$tag) {
+            return [];
+        }
+        return $tag->related_tags;
+    }
+
+    public static function fetchRelatedTags($tag)
+    {
+        $res = StatusHashtag::query()
+            ->select('h2.name', DB::raw('COUNT(*) as related_count'))
+            ->join('status_hashtags as hs2', function ($join) {
+                $join->on('status_hashtags.status_id', '=', 'hs2.status_id')
+                     ->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
+            })
+            ->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
+            ->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
+            ->where('h1.name', '=', $tag)
+            ->groupBy('h2.name')
+            ->orderBy('related_count', 'desc')
+            ->limit(30)
+            ->get();
+
+        return $res;
+    }
+}

app/Services/HashtagService.php

@@ -8,65 +8,80 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\HashtagFollow;
 
-class HashtagService {
+class HashtagService
+{
+    const FOLLOW_KEY = 'pf:services:hashtag:following:v1:';
+    const FOLLOW_PIDS_KEY = 'pf:services:hashtag-follows:v1:';
 
-	const FOLLOW_KEY = 'pf:services:hashtag:following:';
+    public static function get($id)
+    {
+        return Cache::remember('services:hashtag:by_id:' . $id, 3600, function() use($id) {
+            $tag = Hashtag::find($id);
+            if(!$tag) {
+                return [];
+            }
+            return [
+                'name' => $tag->name,
+                'slug' => $tag->slug,
+            ];
+        });
+    }
 
-	public static function get($id)
-	{
-		return Cache::remember('services:hashtag:by_id:' . $id, 3600, function() use($id) {
-			$tag = Hashtag::find($id);
-			if(!$tag) {
-				return [];
-			}
-			return [
-				'name' => $tag->name,
-				'slug' => $tag->slug,
-			];
-		});
-	}
+    public static function count($id)
+    {
+        return Cache::remember('services:hashtag:total-count:by_id:' . $id, 300, function() use($id) {
+            $tag = Hashtag::find($id);
+            return $tag ? $tag->cached_count ?? 0 : 0;
+        });
+    }
 
-	public static function count($id)
-	{
-		return Cache::remember('services:hashtag:public-count:by_id:' . $id, 86400, function() use($id) {
-			return StatusHashtag::whereHashtagId($id)->whereStatusVisibility('public')->count();
-		});
-	}
+    public static function isFollowing($pid, $hid)
+    {
+        $res = Redis::zscore(self::FOLLOW_KEY . $hid, $pid);
+        if($res) {
+            return true;
+        }
 
-	public static function isFollowing($pid, $hid)
-	{
-		$res = Redis::zscore(self::FOLLOW_KEY . $pid, $hid);
-		if($res) {
-			return true;
-		}
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct:' . $pid . ':synced');
+        if(!$synced) {
+            $tags = HashtagFollow::whereProfileId($pid)
+                ->get()
+                ->each(function($tag) use($pid) {
+                    self::follow($pid, $tag->hashtag_id);
+                });
+            Cache::set(self::FOLLOW_KEY . 'acct:' . $pid . ':synced', true, 1209600);
 
-		$synced = Cache::get(self::FOLLOW_KEY . $pid . ':synced');
-		if(!$synced) {
-			$tags = HashtagFollow::whereProfileId($pid)
-				->get()
-				->each(function($tag) use($pid) {
-					self::follow($pid, $tag->hashtag_id);
-				});
-			Cache::set(self::FOLLOW_KEY . $pid . ':synced', true, 1209600);
+            return (bool) Redis::zscore(self::FOLLOW_KEY . $hid, $pid) >= 1;
+        }
 
-			return (bool) Redis::zscore(self::FOLLOW_KEY . $pid, $hid) > 1;
-		}
+        return false;
+    }
 
-		return false;
-	}
+    public static function follow($pid, $hid)
+    {
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
+        return Redis::zadd(self::FOLLOW_KEY . $hid, $pid, $pid);
+    }
 
-	public static function follow($pid, $hid)
-	{
-		return Redis::zadd(self::FOLLOW_KEY . $pid, $hid, $hid);
-	}
+    public static function unfollow($pid, $hid)
+    {
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
+        return Redis::zrem(self::FOLLOW_KEY . $hid, $pid);
+    }
 
-	public static function unfollow($pid, $hid)
-	{
-		return Redis::zrem(self::FOLLOW_KEY . $pid, $hid);
-	}
+    public static function following($hid, $start = 0, $limit = 10)
+    {
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced');
+        if(!$synced) {
+            $tags = HashtagFollow::whereHashtagId($hid)
+                ->get()
+                ->each(function($tag) use($hid) {
+                    self::follow($tag->profile_id, $hid);
+                });
+            Cache::set(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced', true, 1209600);
 
-	public static function following($pid, $start = 0, $limit = 10)
-	{
-		return Redis::zrevrange(self::FOLLOW_KEY . $pid, $start, $limit);
-	}
+            return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
+        }
+        return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
+    }
 }

app/Services/HomeTimelineService.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Follower;
+use App\Status;
+use App\Models\UserDomainBlock;
+
+class HomeTimelineService
+{
+    const CACHE_KEY = 'pf:services:timeline:home:';
+    const FOLLOWER_FEED_POST_LIMIT = 10;
+
+    public static function get($id, $start = 0, $stop = 10)
+    {
+        if($stop > 100) {
+            $stop = 100;
+        }
+
+        return Redis::zrevrange(self::CACHE_KEY . $id, $start, $stop);
+    }
+
+    public static function getRankedMaxId($id, $start = null, $limit = 10)
+    {
+        if(!$start) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, $start, '-inf', [
+            'withscores' => true,
+            'limit' => [1, $limit - 1]
+        ]));
+    }
+
+    public static function getRankedMinId($id, $end = null, $limit = 10)
+    {
+        if(!$end) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, '+inf', $end, [
+            'withscores' => true,
+            'limit' => [0, $limit]
+        ]));
+    }
+
+    public static function add($id, $val)
+    {
+        if(self::count($id) >= 400) {
+            Redis::zpopmin(self::CACHE_KEY . $id);
+        }
+
+        return Redis::zadd(self::CACHE_KEY .$id, $val, $val);
+    }
+
+    public static function rem($id, $val)
+    {
+        return Redis::zrem(self::CACHE_KEY . $id, $val);
+    }
+
+    public static function count($id)
+    {
+        return Redis::zcard(self::CACHE_KEY . $id);
+    }
+
+    public static function warmCache($id, $force = false, $limit = 100, $returnIds = false)
+    {
+        if(self::count($id) == 0 || $force == true) {
+            Redis::del(self::CACHE_KEY . $id);
+            $following = Cache::remember('profile:following:'.$id, 1209600, function() use($id) {
+                $following = Follower::whereProfileId($id)->pluck('following_id');
+                return $following->push($id)->toArray();
+            });
+
+            $minId = SnowflakeService::byDate(now()->subMonths(6));
+
+            $filters = UserFilterService::filters($id);
+
+            if($filters && count($filters)) {
+                $following = array_diff($following, $filters);
+            }
+
+            $domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray();
+
+            $ids = Status::where('id', '>', $minId)
+                ->whereIn('profile_id', $following)
+                ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+                ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+                ->whereIn('visibility',['public', 'unlisted', 'private'])
+                ->orderByDesc('id')
+                ->limit($limit)
+                ->pluck('id');
+
+            foreach($ids as $pid) {
+                $status = StatusService::get($pid, false);
+                if(!$status || !isset($status['account'], $status['url'])) {
+                    continue;
+                }
+                if($domainBlocks && count($domainBlocks)) {
+                    $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+                    if(in_array($domain, $domainBlocks)) {
+                        continue;
+                    }
+                }
+                self::add($id, $pid);
+            }
+
+            return $returnIds ? $ids : 1;
+        }
+        return 0;
+    }
+}

app/Services/InstanceService.php

@@ -120,6 +120,9 @@ class InstanceService
 				$pixels[] = $row;
 			}
 
+			// Free the allocated GdImage object from memory:
+			imagedestroy($image);
+
 			$components_x = 4;
 			$components_y = 4;
 			$blurhash = Blurhash::encode($pixels, $components_x, $components_y);

app/Services/LandingService.php

@@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis;
 use App\Status;
 use App\User;
 use App\Services\AccountService;
+use App\Util\Site\Nodeinfo;
 
 class LandingService
 {
 	public static function get($json = true)
 	{
-		$activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() {
-			return User::select('last_active_at')
-				->where('last_active_at', '>', now()->subMonths(1))
-				->orWhere('created_at', '>', now()->subMonths(1))
-				->count();
-		});
+		$activeMonth = Nodeinfo::activeUsersMonthly();
 
 		$totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() {
 			return User::count();

app/Services/MarkerService.php

@@ -13,7 +13,7 @@ class MarkerService
 		return Cache::get(self::CACHE_KEY . $timeline . ':' . $profileId);
 	}
 
-	public static function set($profileId, $timeline = 'home', $entityId)
+	public static function set($profileId, $timeline = 'home', $entityId = false)
 	{
 		$existing = self::get($profileId, $timeline);
 		$key = self::CACHE_KEY . $timeline . ':' . $profileId;

app/Services/Media/MediaHlsService.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services\Media;
+
+use Storage;
+
+class MediaHlsService
+{
+    public static function allFiles($media)
+    {
+        $path = $media->media_path;
+        if(!$path) { return; }
+        $parts = explode('/', $path);
+        $filename = array_pop($parts);
+        $dir = implode('/', $parts);
+        [$name, $ext] = explode('.', $filename);
+
+        $files = Storage::files($dir);
+
+        return collect($files)
+            ->filter(function($p) use($dir, $name) {
+                return str_starts_with($p, $dir . '/' . $name);
+            })
+            ->values()
+            ->toArray();
+    }
+}

app/Services/MediaService.php

@@ -18,7 +18,7 @@ class MediaService
 
 	public static function get($statusId)
 	{
-		return Cache::remember(self::CACHE_KEY.$statusId, 86400, function() use($statusId) {
+		return Cache::remember(self::CACHE_KEY.$statusId, 21600, function() use($statusId) {
 			$media = Media::whereStatusId($statusId)->orderBy('order')->get();
 			if(!$media) {
 				return [];
@@ -46,7 +46,8 @@ class MediaService
 				$media['orientation'],
 				$media['filter_name'],
 				$media['filter_class'],
-				$media['mime']
+				$media['mime'],
+				$media['hls_manifest']
 			);
 
 			$media['type'] = $mime ? strtolower($mime[0]) : 'unknown';

app/Services/MediaStorageService.php

@@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 use Illuminate\Support\Arr;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
 
 class MediaStorageService {
 
@@ -29,9 +30,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar, $local = false)
+	public static function avatar($avatar, $local = false, $skipRecentCheck = false)
 	{
-		return (new self())->fetchAvatar($avatar, $local);
+		return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
 	}
 
 	public static function head($url)
@@ -86,12 +87,11 @@ class MediaStorageService {
 		$thumbname = array_pop($pt);
 		$storagePath = implode('/', $p);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
-		$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
-		$url = $disk->url($file);
-		$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
-		$thumbUrl = $disk->url($thumbFile);
-		$media->thumbnail_url = $thumbUrl;
+		$url = ResilientMediaStorageService::store($storagePath, $path, $name);
+		if($thumb) {
+			$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
+			$media->thumbnail_url = $thumbUrl;
+		}
 		$media->cdn_url = $url;
 		$media->optimized_url = $url;
 		$media->replicated_at = now();
@@ -183,6 +183,7 @@ class MediaStorageService {
 
 	protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
 	{
+		$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
 		$url = $avatar->remote_url;
 		$driver = $local ? 'local' : config('filesystems.cloud');
 
@@ -206,7 +207,7 @@ class MediaStorageService {
 		$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
 
 		if(!$skipRecentCheck) {
-			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
+			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
 				return;
 			}
 		}
@@ -262,6 +263,7 @@ class MediaStorageService {
 
 		Cache::forget('avatar:' . $avatar->profile_id);
 		AccountService::del($avatar->profile_id);
+		AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
 
 		unlink($tmpName);
 	}

app/Services/NotificationService.php

@@ -12,10 +12,13 @@ use App\Transformer\Api\NotificationTransformer;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
 
 class NotificationService {
 
 	const CACHE_KEY = 'pf:services:notifications:ids:';
+	const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:';
+	const ITEM_CACHE_TTL = 86400;
 	const MASTODON_TYPES = [
 		'follow',
 		'follow_request',
@@ -44,11 +47,22 @@ class NotificationService {
 		return $res;
 	}
 
+	public static function getEpochId($months = 6)
+	{
+		$epoch = Cache::get(self::EPOCH_CACHE_KEY . $months);
+		if(!$epoch) {
+			NotificationEpochUpdatePipeline::dispatch();
+			return 1;
+		}
+		return $epoch;
+	}
+
 	public static function coldGet($id, $start = 0, $stop = 400)
 	{
 		$stop = $stop > 400 ? 400 : $stop;
-		$ids = Notification::whereProfileId($id)
-			->latest()
+		$ids = Notification::where('id', '>', self::getEpochId())
+			->where('profile_id', $id)
+			->orderByDesc('id')
 			->skip($start)
 			->take($stop)
 			->pluck('id');
@@ -227,7 +241,7 @@ class NotificationService {
 
 	public static function getNotification($id)
 	{
-		$notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) {
+		$notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) {
 			$n = Notification::with('item')->find($id);
 
 			if(!$n) {
@@ -259,19 +273,20 @@ class NotificationService {
 
 	public static function setNotification(Notification $notification)
 	{
-		return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
+		return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) {
 			$fractal = new Fractal\Manager();
 			$fractal->setSerializer(new ArraySerializer());
 			$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
 			return $fractal->createData($resource)->toArray();
 		});
-	} 
+	}
 
 	public static function warmCache($id, $stop = 400, $force = false)
 	{
 		if(self::count($id) == 0 || $force == true) {
-			$ids = Notification::whereProfileId($id)
-				->latest()
+			$ids = Notification::where('profile_id', $id)
+				->where('id', '>', self::getEpochId())
+				->orderByDesc('id')
 				->limit($stop)
 				->pluck('id');
 			foreach($ids as $key) {

app/Services/PublicTimelineService.php

@@ -95,7 +95,7 @@ class PublicTimelineService {
 		if(self::count() == 0 || $force == true) {
 			$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
 			Redis::del(self::CACHE_KEY);
-			$minId = SnowflakeService::byDate(now()->subDays(14));
+			$minId = SnowflakeService::byDate(now()->subDays(90));
 			$ids = Status::where('id', '>', $minId)
 				->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id'])
 				->when($hideNsfw, function($q, $hideNsfw) {
@@ -105,9 +105,11 @@ class PublicTimelineService {
 				->whereScope('public')
 				->orderByDesc('id')
 				->limit($limit)
-				->pluck('id');
-			foreach($ids as $id) {
-				self::add($id);
+				->pluck('id', 'profile_id');
+			foreach($ids as $k => $id) {
+                if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) {
+			         self::add($id);
+                }
 			}
 			return 1;
 		}

app/Services/RelationshipService.php

@@ -66,6 +66,14 @@ class RelationshipService
 		return self::get($aid, $tid);
 	}
 
+	public static function forget($aid, $tid)
+	{
+		Cache::forget('pf:services:follower:audience:' . $aid);
+		Cache::forget('pf:services:follower:audience:' . $tid);
+		self::delete($tid, $aid);
+		self::delete($aid, $tid);
+	}
+
 	public static function defaultRelation($tid)
 	{
 		return [

app/Services/ResilientMediaStorageService.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services;
+
+use Storage;
+use Illuminate\Http\File;
+use Exception;
+use GuzzleHttp\Exception\ClientException;
+use Aws\S3\Exception\S3Exception;
+use GuzzleHttp\Exception\ConnectException;
+use League\Flysystem\UnableToWriteFile;
+
+class ResilientMediaStorageService
+{
+    static $attempts = 0;
+
+    public static function store($storagePath, $path, $name)
+    {
+        return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
+            self::handleResilientStore($storagePath, $path, $name) :
+            self::handleStore($storagePath, $path, $name);
+    }
+
+    public static function handleStore($storagePath, $path, $name)
+    {
+        return retry(3, function() use($storagePath, $path, $name) {
+            $baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
+            $disk = Storage::disk($baseDisk);
+            $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            return $disk->url($file);
+        }, random_int(100, 500));
+    }
+
+    public static function handleResilientStore($storagePath, $path, $name)
+    {
+        $attempts = 0;
+        return retry(4, function() use($storagePath, $path, $name, $attempts) {
+            self::$attempts++;
+            usleep(100000);
+            $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
+            try {
+                $disk = Storage::disk($baseDisk);
+                $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
+            return $disk->url($file);
+        }, function (int $attempt, Exception $exception) {
+            return $attempt * 200;
+        });
+    }
+
+    public static function getAltDriver()
+    {
+        $drivers = [];
+        if(config('filesystems.disks.alt-primary.enabled')) {
+            $drivers[] = 'alt-primary';
+        }
+        if(config('filesystems.disks.alt-secondary.enabled')) {
+            $drivers[] = 'alt-secondary';
+        }
+        if(empty($drivers)) {
+            return false;
+        }
+        $key = array_rand($drivers, 1);
+        return $drivers[$key];
+    }
+}

app/Services/SearchApiV2Service.php

@@ -95,7 +95,15 @@ class SearchApiV2Service
         if(substr($webfingerQuery, 0, 1) !== '@') {
             $webfingerQuery = '@' . $webfingerQuery;
         }
-        $banned = InstanceService::getBannedDomains();
+        $banned = InstanceService::getBannedDomains() ?? [];
+        $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
+        if($domainBlocks && count($domainBlocks)) {
+            $banned = array_unique(
+                array_values(
+                    array_merge($banned, $domainBlocks)
+                )
+            );
+        }
         $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like';
         $results = Profile::select('username', 'id', 'followers_count', 'domain')
             ->where('username', $operator, $query)
@@ -172,8 +180,18 @@ class SearchApiV2Service
             'hashtags' => [],
             'statuses' => [],
         ];
+        $user = request()->user();
         $mastodonMode = self::$mastodonMode;
         $query = urldecode($this->query->input('q'));
+        $banned = InstanceService::getBannedDomains();
+        $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
+        if($domainBlocks && count($domainBlocks)) {
+            $banned = array_unique(
+                array_values(
+                    array_merge($banned, $domainBlocks)
+                )
+            );
+        }
         if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) {
             $default['accounts'] = $this->accounts(substr($query, 1));
             return $default;
@@ -197,7 +215,11 @@ class SearchApiV2Service
                 } catch (\Exception $e) {
                     return $default;
                 }
-                if($res && isset($res['id'])) {
+                if($res && isset($res['id'], $res['url'])) {
+                    $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
+                    if(in_array($domain, $banned)) {
+                        return $default;
+                    }
                     $default['accounts'][] = $res;
                     return $default;
                 } else {
@@ -212,6 +234,10 @@ class SearchApiV2Service
                     return $default;
                 }
                 if($res && isset($res['id'])) {
+                    $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
+                    if(in_array($domain, $banned)) {
+                        return $default;
+                    }
                     $default['accounts'][] = $res;
                     return $default;
                 } else {
@@ -221,6 +247,9 @@ class SearchApiV2Service
 
             if($sid = Status::whereUri($query)->first()) {
                 $s = StatusService::get($sid->id, false);
+                if(!$s) {
+                    return $default;
+                }
                 if(in_array($s['visibility'], ['public', 'unlisted'])) {
                     $default['statuses'][] = $s;
                     return $default;
@@ -229,7 +258,7 @@ class SearchApiV2Service
 
             try {
                 $res = ActivityPubFetchService::get($query);
-                $banned = InstanceService::getBannedDomains();
+
                 if($res) {
                     $json = json_decode($res, true);
 

app/Services/StatusHashtagService.php

@@ -84,18 +84,14 @@ class StatusHashtagService {
 
 	public static function statusTags($statusId)
 	{
-		$key = 'pf:services:sh:id:' . $statusId;
+		$status = Status::with('hashtags')->find($statusId);
+		if(!$status) {
+			return [];
+		}
 
-		return Cache::remember($key, 604800, function() use($statusId) {
-			$status = Status::find($statusId);
-			if(!$status) {
-				return [];
-			}
-
-			$fractal = new Fractal\Manager();
-			$fractal->setSerializer(new ArraySerializer());
-			$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
-			return $fractal->createData($resource)->toArray();
-		});
+		$fractal = new Fractal\Manager();
+		$fractal->setSerializer(new ArraySerializer());
+		$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
+		return $fractal->createData($resource)->toArray();
 	}
 }