merge dev

.editorconfig

@@ -1,8 +1,8 @@
 root = true
 
 [*]
+indent_style = space
 indent_size = 4
-indent_style = tab
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true

CHANGELOG.md

@@ -1,6 +1,89 @@
 # Release Notes
 
 ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
+
+### Added
+- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
+- Video WebP2P ([#4713](https://github.com/pixelfed/pixelfed/pull/4713)) ([0405ef12](https://github.com/pixelfed/pixelfed/commit/0405ef12))
+- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
+- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
+- Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
+- Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
+- Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
+- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
+- Added Mutual Followers API endpoint ([33dbbe46](https://github.com/pixelfed/pixelfed/commit/33dbbe46))
+- Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac))
+
+### Federation
+- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
+- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
+
+### Updates
+- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59))
+- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904))
+- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce))
+- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d))
+- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec))
+- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d))
+- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973))
+- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7))
+- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605))
+- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27))
+- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
+- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
+- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
+- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
+- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
+- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
+- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
+- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
+- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
+- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed))
+- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3))
+- Update StatusTransformer, generate autolink on request ([dfe2379b](https://github.com/pixelfed/pixelfed/commit/dfe2379b))
+- Update ComposeModal component, fix multi filter bug and allow media re-ordering before upload/posting ([56e315f6](https://github.com/pixelfed/pixelfed/commit/56e315f6))
+- Update ApiV1Dot1Controller, allow iar rate limits to be configurable ([28a80803](https://github.com/pixelfed/pixelfed/commit/28a80803))
+- Update ApiV1Dot1Controller, add domain to iar redirect ([1f82d47c](https://github.com/pixelfed/pixelfed/commit/1f82d47c))
+- Update ApiV1Dot1Controller, add configurable app confirm rate limit ttl ([4c6a0719](https://github.com/pixelfed/pixelfed/commit/4c6a0719))
+- Update LikePipeline, dispatch to feed queue. Fixes ([#4723](https://github.com/pixelfed/pixelfed/issues/4723)) ([da510089](https://github.com/pixelfed/pixelfed/commit/da510089))
+- Update AccountImport ([5a2d7e3e](https://github.com/pixelfed/pixelfed/commit/5a2d7e3e))
+- Update ImportPostController, fix IG bug with missing spaces between hashtags ([9c24157a](https://github.com/pixelfed/pixelfed/commit/9c24157a))
+- Update ApiV1Controller, fix mutes in home feed ([ddc21714](https://github.com/pixelfed/pixelfed/commit/ddc21714))
+- Update AP helpers, improve preferredUsername validation ([21218c79](https://github.com/pixelfed/pixelfed/commit/21218c79))
+- Update delete pipelines, properly invoke StatusHashtag delete events ([ce54d29c](https://github.com/pixelfed/pixelfed/commit/ce54d29c))
+- Update mail config ([0e431271](https://github.com/pixelfed/pixelfed/commit/0e431271))
+- Update hashtag following ([015b1b80](https://github.com/pixelfed/pixelfed/commit/015b1b80))
+- Update IncrementPostCount job, prevent overlap ([b2c9cc23](https://github.com/pixelfed/pixelfed/commit/b2c9cc23))
+- Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885))
+- Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae))
+- Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008))
+- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
+- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
+- Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
+- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
+- Update StoryApiV1Controller, add self-carousel endpoint. Fixes ([#4352](https://github.com/pixelfed/pixelfed/issues/4352)) ([bcb88d5b](https://github.com/pixelfed/pixelfed/commit/bcb88d5b))
+- Update FollowServiceWarmCache, use more efficient query ([fe9b4c5a](https://github.com/pixelfed/pixelfed/commit/fe9b4c5a))
+- Update HomeFeedPipeline, observe mutes/blocks during fanout ([8548294c](https://github.com/pixelfed/pixelfed/commit/8548294c))
+- Update FederationController, add proper following/follower counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
+- Update FederationController, add proper statuses counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
+- Update Inbox handler, fix missing object_url and uri fields for direct statuses ([a0157fce](https://github.com/pixelfed/pixelfed/commit/a0157fce))
+- Update DirectMessageController, deliver direct delete activities to user inbox instead of sharedInbox ([d848792a](https://github.com/pixelfed/pixelfed/commit/d848792a))
+- Update DirectMessageController, dispatch deliver and delete actions to the job queue ([7f462a80](https://github.com/pixelfed/pixelfed/commit/7f462a80))
+- Update Inbox, improve story attribute collection ([06bee36c](https://github.com/pixelfed/pixelfed/commit/06bee36c))
+- Update DirectMessageController, dispatch local deletes to pipeline ([98186564](https://github.com/pixelfed/pixelfed/commit/98186564))
+- Update StatusPipeline, fix Direct and Story notification deletion ([4c95306f](https://github.com/pixelfed/pixelfed/commit/4c95306f))
+- Update Notifications.vue, fix deprecated DM action links for story activities ([4c3823b0](https://github.com/pixelfed/pixelfed/commit/4c3823b0))
+- Update ComposeModal, fix missing alttext post state ([0a068119](https://github.com/pixelfed/pixelfed/commit/0a068119))
+- Update PhotoAlbumPresenter.vue, fix fullscreen mode ([822e9888](https://github.com/pixelfed/pixelfed/commit/822e9888))
+- Update Timeline.vue, improve CHT pagination ([9c43e7e2](https://github.com/pixelfed/pixelfed/commit/9c43e7e2))
+- Update HomeFeedPipeline, fix StatusService validation ([041c0135](https://github.com/pixelfed/pixelfed/commit/041c0135))
+- Update Inbox, improve tombstone query efficiency ([759a4393](https://github.com/pixelfed/pixelfed/commit/759a4393))
+- Update AccountService, add setLastActive method ([ebbd98e7](https://github.com/pixelfed/pixelfed/commit/ebbd98e7))
+- Update ApiV1Controller, set last_active_at ([b6419545](https://github.com/pixelfed/pixelfed/commit/b6419545))
+- Update AdminShadowFilter, fix deleted profile bug ([a492a95a](https://github.com/pixelfed/pixelfed/commit/a492a95a))
+- Update FollowerService, add $silent param to remove method to more efficently purge relationships ([1664a5bc](https://github.com/pixelfed/pixelfed/commit/1664a5bc))
+- Update AP ProfileTransformer, add published attribute ([adfaa2b1](https://github.com/pixelfed/pixelfed/commit/adfaa2b1))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/AddUserDomainBlock.php

@@ -0,0 +1,106 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\User;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
+use function Laravel\Prompts\text;
+use function Laravel\Prompts\confirm;
+use function Laravel\Prompts\progress;
+
+class AddUserDomainBlock extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:add-user-domain-block';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Apply a domain block to all users';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $domain = text('Enter domain you want to block');
+        $domain = strtolower($domain);
+        $domain = $this->validateDomain($domain);
+        if(!$domain || empty($domain)) {
+            $this->error('Invalid domain');
+            return;
+        }
+        $this->processBlocks($domain);
+        return;
+    }
+
+    protected function validateDomain($domain)
+    {
+        if(!strpos($domain, '.')) {
+            return;
+        }
+
+        if(str_starts_with($domain, 'https://')) {
+            $domain = str_replace('https://', '', $domain);
+        }
+
+        if(str_starts_with($domain, 'http://')) {
+            $domain = str_replace('http://', '', $domain);
+        }
+
+        $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
+
+        $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
+        if(!$valid) {
+            return;
+        }
+
+        if($domain === config('pixelfed.domain.app')) {
+            $this->error('Invalid domain');
+            return;
+        }
+
+        $confirmed = confirm('Are you sure you want to block ' . $domain . '?');
+        if(!$confirmed) {
+            return;
+        }
+
+        return $domain;
+    }
+
+    protected function processBlocks($domain)
+    {
+        DefaultDomainBlock::updateOrCreate([
+            'domain' => $domain
+        ]);
+        progress(
+            label: 'Updating user domain blocks...',
+            steps: User::lazyById(500),
+            callback: fn ($user) => $this->performTask($user, $domain),
+        );
+    }
+
+    protected function performTask($user, $domain)
+    {
+        if(!$user->profile_id || $user->delete_after) {
+            return;
+        }
+
+        if($user->status != null && $user->status != 'disabled') {
+            return;
+        }
+
+        UserDomainBlock::updateOrCreate([
+            'profile_id' => $user->profile_id,
+            'domain' => $domain
+        ]);
+    }
+}

app/Console/Commands/AvatarStorageDeepClean.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Cache;
+use Storage;
+use App\Avatar;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
+
+class AvatarStorageDeepClean extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'avatar:storage-deep-clean';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Cleanup avatar storage';
+
+    protected $shouldKeepRunning = true;
+    protected $counter = 0;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Pixelfed Avatar Deep Cleaner');
+        $this->line(' ');
+        $this->info('    Purge/delete old and outdated avatars from remote accounts');
+        $this->line(' ');
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
+            exit;
+        }
+
+        $start = 0;
+
+        if(!$this->confirm('Are you sure you want to proceed?')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        if(!$this->activeCheck()) {
+            $this->info('Found existing deep cleaning job');
+            if(!$this->confirm('Do you want to continue where you left off?')) {
+                $this->error('Aborting...');
+                exit;
+            } else {
+                $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
+
+                if($start && $start < 1 || $start > PHP_INT_MAX) {
+                    $this->error('Error fetching cached value');
+                    $this->error('Aborting...');
+                    exit;
+                }
+            }
+        }
+
+        $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
+        $bar = $this->output->createProgressBar($count);
+
+        foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
+            usleep(random_int(50, 1000));
+            $this->counter++;
+            $this->handleAvatar($avatar);
+            $bar->advance();
+        }
+        $bar->finish();
+    }
+
+    protected function updateCache($id)
+    {
+        Cache::put('cmd:asdp', $id);
+        if($this->counter % 5 === 0) {
+            Storage::put('avatar-deep-clean.json', $id);
+        }
+    }
+
+    protected function activeCheck()
+    {
+        if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    protected function handleAvatar($avatar)
+    {
+        $this->updateCache($avatar->id);
+        $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
+        $queue = $queues[random_int(0, 7)];
+        AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
+    }
+}

app/Console/Commands/DeleteUserDomainBlock.php

@@ -0,0 +1,96 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\User;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
+use function Laravel\Prompts\text;
+use function Laravel\Prompts\confirm;
+use function Laravel\Prompts\progress;
+
+class DeleteUserDomainBlock extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:delete-user-domain-block';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Remove a domain block for all users';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $domain = text('Enter domain you want to unblock');
+        $domain = strtolower($domain);
+        $domain = $this->validateDomain($domain);
+        if(!$domain || empty($domain)) {
+            $this->error('Invalid domain');
+            return;
+        }
+        $this->processUnblocks($domain);
+        return;
+    }
+
+    protected function validateDomain($domain)
+    {
+        if(!strpos($domain, '.')) {
+            return;
+        }
+
+        if(str_starts_with($domain, 'https://')) {
+            $domain = str_replace('https://', '', $domain);
+        }
+
+        if(str_starts_with($domain, 'http://')) {
+            $domain = str_replace('http://', '', $domain);
+        }
+
+        $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
+
+        $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
+        if(!$valid) {
+            return;
+        }
+
+        if($domain === config('pixelfed.domain.app')) {
+            return;
+        }
+
+        $confirmed = confirm('Are you sure you want to unblock ' . $domain . '?');
+        if(!$confirmed) {
+            return;
+        }
+
+        return $domain;
+    }
+
+    protected function processUnblocks($domain)
+    {
+        DefaultDomainBlock::whereDomain($domain)->delete();
+        if(!UserDomainBlock::whereDomain($domain)->count()) {
+            $this->info('No results found!');
+            return;
+        }
+        progress(
+            label: 'Updating user domain blocks...',
+            steps: UserDomainBlock::whereDomain($domain)->lazyById(500),
+            callback: fn ($domainBlock) => $this->performTask($domainBlock),
+        );
+    }
+
+    protected function performTask($domainBlock)
+    {
+        $domainBlock->deleteQuietly();
+    }
+}

app/Console/Commands/HashtagCachedCountUpdate.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use DB;
+
+class HashtagCachedCountUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-cached-count-update {--limit=100}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update cached counter of hashtags';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $limit = $this->option('limit');
+        $tags = Hashtag::whereNull('cached_count')->limit($limit)->get();
+        $count = count($tags);
+        if(!$count) {
+            return;
+        }
+
+        $bar = $this->output->createProgressBar($count);
+        $bar->start();
+
+        foreach($tags as $tag) {
+            $count = DB::table('status_hashtags')->whereHashtagId($tag->id)->count();
+            if(!$count) {
+                $tag->cached_count = 0;
+                $tag->saveQuietly();
+                $bar->advance();
+                continue;
+            }
+            $tag->cached_count = $count;
+            $tag->saveQuietly();
+            $bar->advance();
+        }
+        $bar->finish();
+        $this->line(' ');
+        return;
+    }
+}

app/Console/Commands/HashtagRelatedGenerate.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+use App\Services\HashtagRelatedService;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use function Laravel\Prompts\multiselect;
+use function Laravel\Prompts\confirm;
+
+class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-related-generate {tag}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'tag' => 'Which hashtag should we generate related tags for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $tag = $this->argument('tag');
+        $hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
+        if(!$hashtag) {
+            $this->error('Hashtag not found, aborting...');
+            exit;
+        }
+
+        $exists = HashtagRelated::whereHashtagId($hashtag->id)->exists();
+
+        if($exists) {
+            $confirmed = confirm('Found existing related tags, do you want to regenerate them?');
+            if(!$confirmed) {
+                $this->error('Aborting...');
+                exit;
+            }
+        }
+
+        $this->info('Looking up #' . $tag . '...');
+
+        $tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
+        if(!$tags || $tags < 100) {
+            $this->error('Not enough posts found to generate related hashtags!');
+            exit;
+        }
+
+        $this->info('Found ' . $tags . ' posts that use that hashtag');
+        $related = collect(HashtagRelatedService::fetchRelatedTags($tag));
+
+        $selected = multiselect(
+            label: 'Which tags do you want to generate?',
+            options: $related->pluck('name'),
+            required: true,
+        );
+
+        $filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
+        $agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
+
+        HashtagRelated::updateOrCreate([
+            'hashtag_id' => $hashtag->id,
+        ], [
+            'related_tags' => array_values($filtered),
+            'agg_score' => $agg_score,
+            'last_calculated_at' => now()
+        ]);
+
+        $this->info('Finished!');
+    }
+}

app/Console/Commands/MediaCloudUrlRewrite.php

@@ -0,0 +1,140 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Media;
+use Cache, Storage;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+
+class MediaCloudUrlRewrite extends Command implements PromptsForMissingInput
+{
+    /**
+    * The name and signature of the console command.
+    *
+    * @var string
+    */
+    protected $signature = 'media:cloud-url-rewrite {oldDomain} {newDomain}';
+
+    /**
+    * Prompt for missing input arguments using the returned questions.
+    *
+    * @return array
+    */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'oldDomain' => 'The old S3 domain',
+            'newDomain' => 'The new S3 domain'
+        ];
+    }
+    /**
+    * The console command description.
+    *
+    * @var string
+    */
+    protected $description = 'Rewrite S3 media urls from local users';
+
+    /**
+    * Execute the console command.
+    */
+    public function handle()
+    {
+        $this->preflightCheck();
+        $this->bootMessage();
+        $this->confirmCloudUrl();
+    }
+
+    protected function preflightCheck()
+    {
+        if(config_cache('pixelfed.cloud_storage') != true) {
+            $this->info('Error: Cloud storage is not enabled!');
+            $this->error('Aborting...');
+            exit;
+        }
+    }
+
+    protected function bootMessage()
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Media Cloud Url Rewrite Tool');
+        $this->info('    ===');
+        $this->info('    Old S3: ' . trim($this->argument('oldDomain')));
+        $this->info('    New S3: ' . trim($this->argument('newDomain')));
+        $this->info(' ');
+    }
+
+    protected function confirmCloudUrl()
+    {
+        $disk = Storage::disk(config('filesystems.cloud'))->url('test');
+        $domain = parse_url($disk, PHP_URL_HOST);
+        if(trim($this->argument('newDomain')) !== $domain) {
+            $this->error('Error: The new S3 domain you entered is not currently configured');
+            exit;
+        }
+
+        if(!$this->confirm('Confirm this is correct')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        $this->updateUrls();
+    }
+
+    protected function updateUrls()
+    {
+        $this->info('Updating urls...');
+        $oldDomain = trim($this->argument('oldDomain'));
+        $newDomain = trim($this->argument('newDomain'));
+        $disk = Storage::disk(config('filesystems.cloud'));
+        $count = Media::whereNotNull('cdn_url')->count();
+        $bar = $this->output->createProgressBar($count);
+        $counter = 0;
+        $bar->start();
+        foreach(Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) {
+            if(strncmp($media->media_path, 'http', 4) === 0) {
+                $bar->advance();
+                continue;
+            }
+            $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST);
+            if($oldDomain != $cdnHost || $newDomain == $cdnHost) {
+                $bar->advance();
+                continue;
+            }
+
+            $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url);
+
+            if($media->thumbnail_url != null) {
+                $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST);
+                if($thumbHost == $oldDomain) {
+                    $thumbUrl = $disk->url($media->thumbnail_path);
+                    $media->thumbnail_url = $thumbUrl;
+                }
+            }
+
+            if($media->optimized_url != null) {
+                $optiHost = parse_url($media->optimized_url, PHP_URL_HOST);
+                if($optiHost == $oldDomain) {
+                    $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url);
+                    $media->optimized_url = $optiUrl;
+                }
+            }
+
+            $media->save();
+            $counter++;
+            $bar->advance();
+        }
+
+        $bar->finish();
+
+        $this->line(' ');
+        $this->info('Finished! Updated ' . $counter . ' total records!');
+        $this->line(' ');
+        $this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
+    }
+}

app/Console/Commands/NotificationEpochUpdate.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
+
+class NotificationEpochUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:notification-epoch-update';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update notification epoch';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        NotificationEpochUpdatePipeline::dispatch();
+    }
+}

app/Console/Commands/UserAdmin.php

@@ -3,16 +3,17 @@
 namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
 use App\User;
 
-class UserAdmin extends Command
+class UserAdmin extends Command implements PromptsForMissingInput
 {
     /**
      * The name and signature of the console command.
      *
      * @var string
      */
-    protected $signature = 'user:admin {id}';
+    protected $signature = 'user:admin {username}';
 
     /**
      * The console command description.
@@ -22,13 +23,15 @@ class UserAdmin extends Command
     protected $description = 'Make a user an admin, or remove admin privileges.';
 
     /**
-     * Create a new command instance.
+     * Prompt for missing input arguments using the returned questions.
      *
-     * @return void
+     * @return array
      */
-    public function __construct()
+    protected function promptForMissingArgumentsUsing()
     {
-        parent::__construct();
+        return [
+            'username' => 'Which username should we toggle admin privileges for?',
+        ];
     }
 
     /**
@@ -38,16 +41,15 @@ class UserAdmin extends Command
      */
     public function handle()
     {
-        $id = $this->argument('id');
-        if(ctype_digit($id) == true) {
-            $user = User::find($id);
-        } else {
+        $id = $this->argument('username');
+
         $user = User::whereUsername($id)->first();
-        }
+
         if(!$user) {
             $this->error('Could not find any user with that username or id.');
             exit;
         }
+
         $this->info('Found username: ' . $user->username);
         $state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?';
         $confirmed = $this->confirm($state);

app/Console/Commands/UserToggle2FA.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use App\User;
+
+class UserToggle2FA extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'user:2fa {username}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Disable two factor authentication for given username';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'username' => 'Which username should we disable 2FA for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $user = User::whereUsername($this->argument('username'))->first();
+
+        if(!$user) {
+            $this->error('Could not find any user with that username');
+            exit;
+        }
+
+        if(!$user->{'2fa_enabled'}) {
+            $this->info('User did not have 2FA enabled!');
+            return;
+        }
+
+        $user->{'2fa_enabled'} = false;
+        $user->{'2fa_secret'} = null;
+        $user->{'2fa_backup_codes'} = null;
+        $user->save();
+
+        $this->info('Successfully disabled 2FA on this account!');
+    }
+}

app/Console/Kernel.php

@@ -43,6 +43,9 @@ class Kernel extends ConsoleKernel
 			$schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37)->onOneServer();
 			$schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32)->onOneServer();
 		}
+
+		$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
+		$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);
 	}
 
 	/**

app/Http/Controllers/Admin/AdminReportController.php

@@ -643,7 +643,7 @@ trait AdminReportController
 				$q->whereNull('admin_seen') :
 				$q->whereNotNull('admin_seen');
 			})
-			->groupBy(['object_id', 'object_type'])
+			->groupBy(['id', 'object_id', 'object_type', 'profile_id'])
 			->cursorPaginate(6)
 			->withQueryString()
 		);

app/Http/Controllers/AdminShadowFilterController.php

@@ -0,0 +1,123 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+use App\Models\AdminShadowFilter;
+use App\Profile;
+use App\Services\AccountService;
+use App\Services\AdminShadowFilterService;
+
+class AdminShadowFilterController extends Controller
+{
+    public function __construct()
+    {
+        $this->middleware(['auth','admin']);
+    }
+
+    public function home(Request $request)
+    {
+        $filter = $request->input('filter');
+        $searchQuery = $request->input('q');
+        $filters = AdminShadowFilter::whereHas('profile')
+        ->when($filter, function($q, $filter) {
+            if($filter == 'all') {
+                return $q;
+            } else if($filter == 'inactive') {
+                return $q->whereActive(false);
+            } else {
+                return $q;
+            }
+        }, function($q, $filter) {
+            return $q->whereActive(true);
+        })
+        ->when($searchQuery, function($q, $searchQuery) {
+            $ids = Profile::where('username', 'like', '%' . $searchQuery . '%')
+                ->limit(100)
+                ->pluck('id')
+                ->toArray();
+            return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids);
+        })
+        ->latest()
+        ->paginate(10)
+        ->withQueryString();
+
+        return view('admin.asf.home', compact('filters'));
+    }
+
+    public function create(Request $request)
+    {
+        return view('admin.asf.create');
+    }
+
+    public function edit(Request $request, $id)
+    {
+        $filter = AdminShadowFilter::findOrFail($id);
+        $profile = AccountService::get($filter->item_id);
+        return view('admin.asf.edit', compact('filter', 'profile'));
+    }
+
+    public function store(Request $request)
+    {
+        $this->validate($request, [
+            'username' => 'required',
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $profile = Profile::whereUsername($request->input('username'))->first();
+
+        if(!$profile) {
+            return back()->withErrors(['Invalid account']);
+        }
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active') && $request->has('hide_from_public_feeds');
+
+        AdminShadowFilter::updateOrCreate([
+            'item_id' => $profile->id,
+            'item_type' => get_class($profile)
+        ], [
+            'is_local' => $profile->domain === null,
+            'note' => $request->input('note'),
+            'hide_from_public_feeds' => $request->has('hide_from_public_feeds'),
+            'admin_id' => $request->user()->profile_id,
+            'active' => $active
+        ]);
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+
+    public function storeEdit(Request $request, $id)
+    {
+        $this->validate($request, [
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $filter = AdminShadowFilter::findOrFail($id);
+
+        $profile = Profile::findOrFail($filter->item_id);
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active');
+        $filter->active = $active;
+        $filter->hide_from_public_feeds = $request->has('hide_from_public_feeds');
+        $filter->note = $request->input('note');
+        $filter->save();
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+}

app/Http/Controllers/Api/ApiV1Controller.php

@@ -31,6 +31,7 @@ use App\{
     UserSetting,
     UserFilter,
 };
+use App\Models\UserDomainBlock;
 use League\Fractal;
 use App\Transformer\Api\Mastodon\v1\{
     AccountTransformer,
@@ -71,6 +72,8 @@ use App\Services\{
     CollectionService,
     FollowerService,
     HashtagService,
+    HashtagFollowService,
+    HomeTimelineService,
     InstanceService,
     LikeService,
     NetworkTimelineService,
@@ -102,6 +105,8 @@ use Illuminate\Support\Facades\RateLimiter;
 use Purify;
 use Carbon\Carbon;
 use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
 
 class ApiV1Controller extends Controller
 {
@@ -185,6 +190,7 @@ class ApiV1Controller extends Controller
 
         abort_if(!$user, 403);
         abort_if($user->status != null, 403);
+        AccountService::setLastActive($user->id);
 
         $res = $request->has(self::PF_API_ENTITY_KEY) ? AccountService::get($user->profile_id) : AccountService::getMastodon($user->profile_id);
 
@@ -243,6 +249,7 @@ class ApiV1Controller extends Controller
         ]);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
         $profile = $user->profile;
         $settings = $user->settings;
 
@@ -750,6 +757,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
 
         $target = Profile::where('id', '!=', $user->profile_id)
             ->whereNull('status')
@@ -834,6 +842,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
 
         $target = Profile::where('id', '!=', $user->profile_id)
             ->whereNull('status')
@@ -937,6 +946,7 @@ class ApiV1Controller extends Controller
         ]);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
         $query = $request->input('q');
         $limit = $request->input('limit') ?? 20;
         $resolve = (bool) $request->input('resolve', false);
@@ -1009,6 +1019,7 @@ class ApiV1Controller extends Controller
 
         $user = $request->user();
         $pid = $user->profile_id ?? $user->profile->id;
+        AccountService::setLastActive($user->id);
 
         if(intval($id) === intval($pid)) {
             abort(400, 'You cannot block yourself');
@@ -1101,6 +1112,7 @@ class ApiV1Controller extends Controller
 
         $user = $request->user();
         $pid = $user->profile_id ?? $user->profile->id;
+        AccountService::setLastActive($user->id);
 
         if(intval($id) === intval($pid)) {
             abort(400, 'You cannot unblock yourself');
@@ -1233,6 +1245,8 @@ class ApiV1Controller extends Controller
 
         $user = $request->user();
 
+        AccountService::setLastActive($user->id);
+
         $status = StatusService::getMastodon($id, false);
 
         abort_unless($status, 400);
@@ -1271,7 +1285,7 @@ class ApiV1Controller extends Controller
             Status::findOrFail($status['id'])->update([
                 'likes_count' => ($status['favourites_count'] ?? 0) + 1
             ]);
-			LikePipeline::dispatch($like);
+            LikePipeline::dispatch($like)->onQueue('feed');
         }
 
         $status['favourited'] = true;
@@ -1292,6 +1306,8 @@ class ApiV1Controller extends Controller
 
         $user = $request->user();
 
+        AccountService::setLastActive($user->id);
+
         $status = Status::findOrFail($id);
 
         if(intval($status->profile_id) !== intval($user->profile_id)) {
@@ -1308,7 +1324,7 @@ class ApiV1Controller extends Controller
 
         if($like) {
             $like->forceDelete();
-			$status->likes_count = $status->likes()->count();
+            $status->likes_count = $status->likes_count > 1 ? $status->likes_count - 1 : 0;
             $status->save();
         }
 
@@ -1607,6 +1623,7 @@ class ApiV1Controller extends Controller
         ]);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
 
         if($user->last_active_at == null) {
             return [];
@@ -1728,6 +1745,7 @@ class ApiV1Controller extends Controller
         ]);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
 
         $media = Media::whereUserId($user->id)
             ->whereProfileId($user->profile_id)
@@ -1774,6 +1792,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
 
         $media = Media::whereUserId($user->id)
             ->whereNull('status_id')
@@ -1817,6 +1836,8 @@ class ApiV1Controller extends Controller
             return [];
         }
 
+        AccountService::setLastActive($user->id);
+
         if(empty($request->file('file'))) {
             return response('', 422);
         }
@@ -2061,6 +2082,8 @@ class ApiV1Controller extends Controller
             'min_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
             'max_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
             'since_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
+            'types[]' => 'sometimes|array',
+            'type' => 'sometimes|string|in:mention,reblog,follow,favourite'
         ]);
 
         $pid = $request->user()->profile_id;
@@ -2074,29 +2097,29 @@ class ApiV1Controller extends Controller
             $min = 1;
         }
 
+        $types = $request->input('types');
+
         $maxId = null;
         $minId = null;
+        AccountService::setLastActive($request->user()->id);
 
-		if($max) {
-			$res = NotificationService::getMaxMastodon($pid, $max, $limit);
-			$ids = NotificationService::getRankedMaxId($pid, $max, $limit);
+        $res = $max ?
+            NotificationService::getMaxMastodon($pid, $max, $limit) :
+            NotificationService::getMinMastodon($pid, $min ?? $since, $limit);
+        $ids = $max ?
+            NotificationService::getRankedMaxId($pid, $max, $limit) :
+            NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
         if(!empty($ids)) {
             $maxId = max($ids);
             $minId = min($ids);
         }
-		} else {
-			$res = NotificationService::getMinMastodon($pid, $min ?? $since, $limit);
-			$ids = NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
-			if(!empty($ids)) {
-				$maxId = max($ids);
-				$minId = min($ids);
-			}
-		}
 
-		if(empty($res) && !Cache::has('pf:services:notifications:hasSynced:'.$pid)) {
+        if(empty($res)) {
+            if(!Cache::has('pf:services:notifications:hasSynced:'.$pid)) {
                 Cache::put('pf:services:notifications:hasSynced:'.$pid, 1, 1209600);
                 NotificationService::warmCache($pid, 400, true);
             }
+        }
 
         $baseUrl = config('app.url') . '/api/v1/notifications?limit=' . $limit . '&';
 
@@ -2132,7 +2155,7 @@ class ApiV1Controller extends Controller
             'page'        => 'sometimes|integer|max:40',
             'min_id'      => 'sometimes|integer|min:0|max:' . PHP_INT_MAX,
             'max_id'      => 'sometimes|integer|min:0|max:' . PHP_INT_MAX,
-		  'limit'       => 'sometimes|integer|min:1|max:100',
+            'limit'       => 'sometimes|integer|min:1|max:40',
             'include_reblogs' => 'sometimes',
         ]);
 
@@ -2142,19 +2165,100 @@ class ApiV1Controller extends Controller
         $max = $request->input('max_id');
         $limit = $request->input('limit') ?? 20;
         $pid = $request->user()->profile_id;
-        $includeReblogs = $request->filled('include_reblogs');
+        $includeReblogs = $request->filled('include_reblogs') ? $request->boolean('include_reblogs') : false;
         $nullFields = $includeReblogs ?
         ['in_reply_to_id'] :
         ['in_reply_to_id', 'reblog_of_id'];
         $inTypes = $includeReblogs ?
         ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album', 'share'] :
         ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'];
+        AccountService::setLastActive($request->user()->id);
+
+        if(config('exp.cached_home_timeline')) {
+            $paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50;
+            if($min || $max) {
+                if($request->has('min_id')) {
+                    $res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
+                } else {
+                    $res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
+                }
+            } else {
+                $res = HomeTimelineService::get($pid, 0, $paddedLimit);
+            }
+
+            if(!$res) {
+                $res = Cache::has('pf:services:apiv1:home:cached:coldbootcheck:' . $pid);
+                if(!$res) {
+                    Cache::set('pf:services:apiv1:home:cached:coldbootcheck:' . $pid, 1, 86400);
+                    FeedWarmCachePipeline::dispatchSync($pid);
+                    return response()->json([], 206);
+                } else {
+                    Cache::set('pf:services:apiv1:home:cached:coldbootcheck:' . $pid, 1, 86400);
+                    return response()->json([], 206);
+                }
+            }
+
+            $res = collect($res)
+            ->map(function($id) use($napi) {
+                return $napi ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
+            })
+            ->filter(function($res) {
+                return $res && isset($res['account']);
+            })
+            ->filter(function($s) use($includeReblogs) {
+                return $includeReblogs ? true : $s['reblog'] == null;
+            })
+            ->take($limit)
+            ->map(function($status) use($pid) {
+                if($pid) {
+                    $status['favourited'] = (bool) LikeService::liked($pid, $status['id']);
+                    $status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
+                    $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
+                }
+                return $status;
+            })
+            ->values();
+
+            $baseUrl = config('app.url') . '/api/v1/timelines/home?limit=' . $limit . '&';
+            $minId = $res->map(function($s) {
+                return ['id' => $s['id']];
+            })->min('id');
+            $maxId = $res->map(function($s) {
+                return ['id' => $s['id']];
+            })->max('id');
+
+            if($minId == $maxId) {
+                $minId = null;
+            }
+
+            if($maxId) {
+                $link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next"';
+            }
+
+            if($minId) {
+                $link = '<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
+            }
+
+            if($maxId && $minId) {
+                $link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next",<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
+            }
+
+            $headers = isset($link) ? ['Link' => $link] : [];
+            return $this->json($res->toArray(), 200, $headers);
+        }
 
         $following = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
             $following = Follower::whereProfileId($pid)->pluck('following_id');
             return $following->push($pid)->toArray();
         });
 
+        $muted = UserFilterService::mutes($pid);
+
+        if($muted && count($muted)) {
+            $following = array_diff($following, $muted);
+        }
+
+
         if($min || $max) {
             $dir = $min ? '>' : '<';
             $id = $min ?? $max;
@@ -2315,10 +2419,13 @@ class ApiV1Controller extends Controller
         $max = $request->input('max_id');
         $limit = $request->input('limit') ?? 20;
         $user = $request->user();
-		$remote = ($request->has('remote') && $request->input('remote') == true) || ($request->filled('local') && $request->input('local') != true);
+        $remote = $request->has('remote');
+        $local = $request->has('local');
         $filtered = $user ? UserFilterService::filters($user->profile_id) : [];
+        AccountService::setLastActive($user->id);
+        $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
 
-        if((!$request->has('local') || $remote) && config('instance.timeline.network.cached')) {
+        if($remote && config('instance.timeline.network.cached')) {
             Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
                 if(NetworkTimelineService::count() == 0) {
                     NetworkTimelineService::warmCache(true, config('instance.timeline.network.cache_dropoff'));
@@ -2332,7 +2439,9 @@ class ApiV1Controller extends Controller
             } else {
                 $feed = NetworkTimelineService::get(0, $limit + 5);
             }
-        } else {
+        }
+
+        if($local || !$remote && !$local) {
             Cache::remember('api:v1:timelines:public:cache_check', 10368000, function() {
                 if(PublicTimelineService::count() == 0) {
                     PublicTimelineService::warmCache(true, 400);
@@ -2389,6 +2498,13 @@ class ApiV1Controller extends Controller
         ->filter(function($s) use($filtered) {
             return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
         })
+        ->filter(function($s) use($domainBlocks) {
+            if(!$domainBlocks || !count($domainBlocks)) {
+                return $s;
+            }
+            $domain = strtolower(parse_url($s['url'], PHP_URL_HOST));
+            return !in_array($domain, $domainBlocks);
+        })
         ->take($limit)
         ->values();
 
@@ -2506,8 +2622,8 @@ class ApiV1Controller extends Controller
     public function statusById(Request $request, $id)
     {
         abort_if(!$request->user(), 403);
-
-		$user = $request->user();
+        AccountService::setLastActive($request->user()->id);
+        $pid = $request->user()->profile_id;
 
         $res = $request->has(self::PF_API_ENTITY_KEY) ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
         if(!$res || !isset($res['visibility'])) {
@@ -2517,17 +2633,23 @@ class ApiV1Controller extends Controller
         $scope = $res['visibility'];
         if(!in_array($scope, ['public', 'unlisted'])) {
             if($scope === 'private') {
-				if(intval($res['account']['id']) !== intval($user->profile_id)) {
-					abort_unless(FollowerService::follows($user->profile_id, $res['account']['id']), 403);
+                if(intval($res['account']['id']) !== intval($pid)) {
+                    abort_unless(FollowerService::follows($pid, $res['account']['id']), 403);
                 }
             } else {
                 abort(400, 'Invalid request');
             }
         }
 
-		$res['favourited'] = LikeService::liked($user->profile_id, $res['id']);
-		$res['reblogged'] = ReblogService::get($user->profile_id, $res['id']);
-		$res['bookmarked'] = BookmarkService::get($user->profile_id, $res['id']);
+        if(!empty($res['reblog']) && isset($res['reblog']['id'])) {
+            $res['reblog']['favourited'] = (bool) LikeService::liked($pid, $res['reblog']['id']);
+            $res['reblog']['reblogged'] = (bool) ReblogService::get($pid, $res['reblog']['id']);
+            $res['reblog']['bookmarked'] = BookmarkService::get($pid, $res['reblog']['id']);
+        }
+
+        $res['favourited'] = LikeService::liked($pid, $res['id']);
+        $res['reblogged'] = ReblogService::get($pid, $res['id']);
+        $res['bookmarked'] = BookmarkService::get($pid, $res['id']);
 
         return $this->json($res);
     }
@@ -2544,6 +2666,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
         $pid = $user->profile_id;
         $status = StatusService::getMastodon($id, false);
 
@@ -3014,7 +3137,7 @@ class ApiV1Controller extends Controller
     public function statusDelete(Request $request, $id)
     {
         abort_if(!$request->user(), 403);
-
+        AccountService::setLastActive($request->user()->id);
         $status = Status::whereProfileId($request->user()->profile->id)
         ->findOrFail($id);
 
@@ -3042,6 +3165,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
         $status = Status::whereScope('public')->findOrFail($id);
 
         if(intval($status->profile_id) !== intval($user->profile_id)) {
@@ -3088,6 +3212,7 @@ class ApiV1Controller extends Controller
         abort_if(!$request->user(), 403);
 
         $user = $request->user();
+        AccountService::setLastActive($user->id);
         $status = Status::whereScope('public')->findOrFail($id);
 
         if(intval($status->profile_id) !== intval($user->profile_id)) {
@@ -3160,6 +3285,7 @@ class ApiV1Controller extends Controller
         $limit = $request->input('limit', 20);
         $onlyMedia = $request->input('only_media', true);
         $pe = $request->has(self::PF_API_ENTITY_KEY);
+        $pid = $request->user()->profile_id;
 
         if($min || $max) {
             $minMax = SnowflakeService::byDate(now()->subMonths(6));
@@ -3171,7 +3297,8 @@ class ApiV1Controller extends Controller
             }
         }
 
-		$filters = UserFilterService::filters($request->user()->profile_id);
+        $filters = UserFilterService::filters($pid);
+        $domainBlocks = UserFilterService::domainBlocks($pid);
 
         if(!$min && !$max) {
             $id = 1;
@@ -3197,10 +3324,11 @@ class ApiV1Controller extends Controller
                 if($onlyMedia && !isset($i['media_attachments']) || !count($i['media_attachments'])) {
                     return false;
                 }
-				return $i && isset($i['account']);
+                return $i && isset($i['account'], $i['url']);
             })
-			->filter(function($i) use($filters) {
-				return !in_array($i['account']['id'], $filters);
+            ->filter(function($i) use($filters, $domainBlocks) {
+                $domain = strtolower(parse_url($i['url'], PHP_URL_HOST));
+                return !in_array($i['account']['id'], $filters) && !in_array($domain, $domainBlocks);
             })
             ->values()
             ->toArray();
@@ -3503,38 +3631,31 @@ class ApiV1Controller extends Controller
 
         $pid = $request->user()->profile_id;
 
-		$ids = Cache::remember('api:v1.1:discover:accounts:popular', 86400, function() {
+        $ids = Cache::remember('api:v1.1:discover:accounts:popular', 3600, function() {
             return DB::table('profiles')
             ->where('is_private', false)
             ->whereNull('status')
             ->orderByDesc('profiles.followers_count')
-			->limit(20)
+            ->limit(30)
             ->get();
         });
-
+        $filters = UserFilterService::filters($pid);
         $ids = $ids->map(function($profile) {
             return AccountService::get($profile->id, true);
         })
         ->filter(function($profile) use($pid) {
-			return $profile && isset($profile['id']);
+            return $profile && isset($profile['id'], $profile['locked']) && !$profile['locked'];
         })
         ->filter(function($profile) use($pid) {
             return $profile['id'] != $pid;
         })
-        ->map(function($profile) {
-            $ids = collect(ProfileStatusService::get($profile['id'], 0, 9))
-                ->map(function($id) {
-                    return StatusService::get($id, true);
+        ->filter(function($profile) use($pid) {
+            return !FollowerService::follows($pid, $profile['id'], true);
         })
-                ->filter(function($post) {
-                    return $post && isset($post['id']);
+        ->filter(function($profile) use($filters) {
+            return !in_array($profile['id'], $filters);
         })
-                ->take(3)
-                ->values();
-            $profile['recent_posts'] = $ids;
-            return $profile;
-        })
-		->take(6)
+        ->take(16)
         ->values();
 
         return $this->json($ids);
@@ -3633,168 +3754,4 @@ class ApiV1Controller extends Controller
 
         return $this->json([]);
     }
-
-	/**
-	* GET /api/v1/followed_tags
-	*
-	*
-	* @return array
-	*/
-	public function getFollowedTags(Request $request)
-	{
-		abort_if(!$request->user(), 403);
-
-		$account = AccountService::get($request->user()->profile_id);
-
-		$this->validate($request, [
-			'cursor' => 'sometimes',
-			'limit' => 'sometimes|integer|min:1|max:200'
-		]);
-		$limit = $request->input('limit', 100);
-
-		$res = HashtagFollow::whereProfileId($account['id'])
-			->orderByDesc('id')
-			->cursorPaginate($limit)->withQueryString();
-
-		$pagination = false;
-		$prevPage = $res->nextPageUrl();
-		$nextPage = $res->previousPageUrl();
-		if($nextPage && $prevPage) {
-			$pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
-		} else if($nextPage && !$prevPage) {
-			$pagination = '<' . $nextPage . '>; rel="next"';
-		} else if(!$nextPage && $prevPage) {
-			$pagination = '<' . $prevPage . '>; rel="prev"';
-		}
-
-		if($pagination) {
-			return response()->json(FollowedTagResource::collection($res)->collection)
-				->header('Link', $pagination);
-		}
-		return response()->json(FollowedTagResource::collection($res)->collection);
-	}
-
-	/**
-	* POST /api/v1/tags/:id/follow
-	*
-	*
-	* @return object
-	*/
-	public function followHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		abort_if(!$tag, 422, 'Unknown hashtag');
-
-		abort_if(
-			HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
-			422,
-			'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
-		);
-
-		$follows = HashtagFollow::updateOrCreate(
-			[
-				'profile_id' => $account['id'],
-				'hashtag_id' => $tag->id
-			],
-			[
-				'user_id' => $request->user()->id
-			]
-		);
-
-		HashtagService::follow($pid, $tag->id);
-
-		return response()->json(FollowedTagResource::make($follows)->toArray($request));
-	}
-
-	/**
-	* POST /api/v1/tags/:id/unfollow
-	*
-	*
-	* @return object
-	*/
-	public function unfollowHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		abort_if(!$tag, 422, 'Unknown hashtag');
-
-		$follows = HashtagFollow::whereProfileId($pid)
-			->whereHashtagId($tag->id)
-			->first();
-
-		if(!$follows) {
-			return [
-				'name' => $tag->name,
-				'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
-				'history' => [],
-				'following' => false
-			];
-		}
-
-		if($follows) {
-			HashtagService::unfollow($pid, $tag->id);
-			$follows->delete();
-		}
-
-		$res = FollowedTagResource::make($follows)->toArray($request);
-		$res['following'] = false;
-		return response()->json($res);
-	}
-
-	/**
-	* GET /api/v1/tags/:id
-	*
-	*
-	* @return object
-	*/
-	public function getHashtag(Request $request, $id)
-	{
-		abort_if(!$request->user(), 403);
-
-		$pid = $request->user()->profile_id;
-		$account = AccountService::get($pid);
-		$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
-		$tag = Hashtag::where('name', $operator, $id)
-			->orWhere('slug', $operator, $id)
-			->first();
-
-		if(!$tag) {
-			return [
-				'name' => $id,
-				'url' => config('app.url') . '/i/web/hashtag/' . $id,
-				'history' => [],
-				'following' => false
-			];
-		}
-
-		$res = [
-			'name' => $tag->name,
-			'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
-			'history' => [],
-			'following' => HashtagService::isFollowing($pid, $tag->id)
-		];
-
-		if($request->has(self::PF_API_ENTITY_KEY)) {
-			$res['count'] = HashtagService::count($tag->id);
-		}
-
-		return $this->json($res);
-	}
 }

app/Http/Controllers/Api/ApiV1Dot1Controller.php

@@ -11,6 +11,7 @@ use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 use App\AccountLog;
 use App\EmailVerification;
+use App\Follower;
 use App\Place;
 use App\Status;
 use App\Report;
@@ -19,8 +20,11 @@ use App\StatusArchived;
 use App\User;
 use App\UserSetting;
 use App\Services\AccountService;
+use App\Services\FollowerService;
 use App\Services\StatusService;
 use App\Services\ProfileStatusService;
+use App\Services\LikeService;
+use App\Services\ReblogService;
 use App\Services\PublicTimelineService;
 use App\Services\NetworkTimelineService;
 use App\Util\Lexer\RestrictedNames;
@@ -470,7 +474,7 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), 3, function(){}, 1800);
+		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), config('pixelfed.app_registration_rate_limit_attempts', 3), function(){}, config('pixelfed.app_registration_rate_limit_decay', 1800));
 		abort_if(!$rl, 400, 'Too many requests');
 
 		$this->validate($request, [
@@ -543,10 +547,10 @@ class ApiV1Dot1Controller extends Controller
 		$user->password = Hash::make($password);
 		$user->register_source = 'app';
 		$user->app_register_ip = $request->ip();
-		$user->app_register_token = Str::random(32);
+		$user->app_register_token = Str::random(40);
 		$user->save();
 
-		$rtoken = Str::random(mt_rand(64, 70));
+		$rtoken = Str::random(64);
 
 		$verify = new EmailVerification();
 		$verify->user_id = $user->id;
@@ -555,7 +559,12 @@ class ApiV1Dot1Controller extends Controller
 		$verify->random_token = $rtoken;
 		$verify->save();
 
-		$appUrl = url('/api/v1.1/auth/iarer?ut=' . $user->app_register_token . '&rt=' . $rtoken);
+		$params = http_build_query([
+			'ut' => $user->app_register_token,
+			'rt' => $rtoken,
+			'ea' => base64_encode($user->email)
+		]);
+		$appUrl = url('/api/v1.1/auth/iarer?'. $params);
 
 		Mail::to($user->email)->send(new ConfirmAppEmail($verify, $appUrl));
 
@@ -568,14 +577,19 @@ class ApiV1Dot1Controller extends Controller
 	{
 		$this->validate($request, [
 			'ut' => 'required',
-			'rt' => 'required'
+			'rt' => 'required',
+			'ea' => 'required'
 		]);
-		if(config('pixelfed.bouncer.cloud_ips.ban_signups')) {
-			abort_if(BouncerService::checkIp($request->ip()), 404);
-		}
 		$ut = $request->input('ut');
 		$rt = $request->input('rt');
-		$url = 'pixelfed://confirm-account/'. $ut . '?rt=' . $rt;
+		$ea = $request->input('ea');
+		$params = http_build_query([
+			'ut' => $ut,
+			'rt' => $rt,
+			'domain' => config('pixelfed.domain.app'),
+			'ea' => $ea
+		]);
+		$url = 'pixelfed://confirm-account/'. $ut . '?' . $params;
 		return redirect()->away($url);
 	}
 
@@ -589,8 +603,8 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), 10, function(){}, 1800);
-		abort_if(!$rl, 400, 'Too many requests');
+		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), config('pixelfed.app_registration_confirm_rate_limit_attempts', 20), function(){}, config('pixelfed.app_registration_confirm_rate_limit_decay', 1800));
+		abort_if(!$rl, 429, 'Too many requests');
 
 		$this->validate($request, [
 			'user_token' => 'required',
@@ -884,4 +898,19 @@ class ApiV1Dot1Controller extends Controller
 
         return [200];
     }
+
+    public function getMutualAccounts(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+        $account = AccountService::get($id, true);
+        if(!$account || !isset($account['id'])) { return []; }
+        $res = collect(FollowerService::mutualAccounts($request->user()->profile_id, $id))
+            ->map(function($accountId) {
+                return AccountService::get($accountId, true);
+            })
+            ->filter()
+            ->take(24)
+            ->values();
+        return $this->json($res);
+    }
 }

app/Http/Controllers/Api/ApiV2Controller.php

@@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{
 use App\Transformer\Api\{
 	RelationshipTransformer,
 };
+use App\Util\Site\Nodeinfo;
 
 class ApiV2Controller extends Controller
 {
@@ -77,12 +78,7 @@ class ApiV2Controller extends Controller
 			'description' => config_cache('app.short_description'),
 			'usage' => [
 				'users' => [
-					'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() {
-						return User::select('last_active_at', 'created_at')
-							->where('last_active_at', '>', now()->subMonths(1))
-							->orWhere('created_at', '>', now()->subMonths(1))
-							->count();
-					})
+					'active_month' => (int) Nodeinfo::activeUsersMonthly()
 				]
 			],
 			'thumbnail' => [

app/Http/Controllers/Api/V1/DomainBlockController.php

@@ -0,0 +1,118 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Models\UserDomainBlock;
+use App\Util\ActivityPub\Helpers;
+use App\Services\UserFilterService;
+use Illuminate\Bus\Batch;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\Cache;
+use App\Jobs\HomeFeedPipeline\FeedRemoveDomainPipeline;
+use App\Jobs\ProfilePipeline\ProfilePurgeNotificationsByDomain;
+use App\Jobs\ProfilePipeline\ProfilePurgeFollowersByDomain;
+
+class DomainBlockController extends Controller
+{
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    public function index(Request $request)
+    {
+        abort_unless($request->user(), 403);
+        $this->validate($request, [
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+        $id = $request->user()->profile_id;
+        $filters = UserDomainBlock::whereProfileId($id)->orderByDesc('id')->cursorPaginate($limit);
+        $links = null;
+        $headers = [];
+
+        if($filters->nextCursor()) {
+            $links .= '<'.$filters->nextPageUrl().'&limit='.$limit.'>; rel="next"';
+        }
+
+        if($filters->previousCursor()) {
+            if($links != null) {
+                $links .= ', ';
+            }
+            $links .= '<'.$filters->previousPageUrl().'&limit='.$limit.'>; rel="prev"';
+        }
+
+        if($links) {
+            $headers = ['Link' => $links];
+        }
+        return $this->json($filters->pluck('domain'), 200, $headers);
+    }
+
+    public function store(Request $request)
+    {
+        abort_unless($request->user(), 403);
+
+        $this->validate($request, [
+            'domain' => 'required|active_url|min:1|max:120'
+        ]);
+
+        $pid = $request->user()->profile_id;
+
+        $domain = trim($request->input('domain'));
+
+        if(Helpers::validateUrl($domain) == false) {
+            return abort(500, 'Invalid domain or already blocked by server admins');
+        }
+
+        $domain = strtolower(parse_url($domain, PHP_URL_HOST));
+
+        abort_if(config_cache('pixelfed.domain.app') == $domain, 400, 'Cannot ban your own server');
+
+        $existingCount = UserDomainBlock::whereProfileId($pid)->count();
+        $maxLimit = config('instance.user_filters.max_domain_blocks');
+        $errorMsg =  __('profile.block.domain.max', ['max' => $maxLimit]);
+
+        abort_if($existingCount >= $maxLimit, 400, $errorMsg);
+
+        $block = UserDomainBlock::updateOrCreate([
+            'profile_id' => $pid,
+            'domain' => $domain
+        ]);
+
+        if($block->wasRecentlyCreated) {
+            Bus::batch([
+                [
+                    new FeedRemoveDomainPipeline($pid, $domain),
+                    new ProfilePurgeNotificationsByDomain($pid, $domain),
+                    new ProfilePurgeFollowersByDomain($pid, $domain)
+                ]
+            ])->allowFailures()->onQueue('feed')->dispatch();
+
+            Cache::forget('profile:following:' . $pid);
+            UserFilterService::domainBlocks($pid, true);
+        }
+
+        return $this->json([]);
+    }
+
+    public function delete(Request $request)
+    {
+        abort_unless($request->user(), 403);
+
+        $this->validate($request, [
+            'domain' => 'required|min:1|max:120'
+        ]);
+
+        $pid = $request->user()->profile_id;
+
+        $domain = strtolower(trim($request->input('domain')));
+
+        $filters = UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->delete();
+
+        UserFilterService::domainBlocks($pid, true);
+
+        return $this->json([]);
+    }
+}

app/Http/Controllers/Api/V1/TagsController.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Hashtag;
+use App\HashtagFollow;
+use App\StatusHashtag;
+use App\Services\AccountService;
+use App\Services\HashtagService;
+use App\Services\HashtagFollowService;
+use App\Services\HashtagRelatedService;
+use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+
+class TagsController extends Controller
+{
+    const PF_API_ENTITY_KEY = "_pe";
+
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    /**
+    * GET /api/v1/tags/:id/related
+    *
+    *
+    * @return array
+    */
+    public function relatedTags(Request $request, $tag)
+    {
+        abort_unless($request->user(), 403);
+        $tag = Hashtag::whereSlug($tag)->firstOrFail();
+        return HashtagRelatedService::get($tag->id);
+    }
+
+    /**
+    * POST /api/v1/tags/:id/follow
+    *
+    *
+    * @return object
+    */
+    public function followHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        abort_if(
+            HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
+            422,
+            'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
+        );
+
+        $follows = HashtagFollow::updateOrCreate(
+            [
+                'profile_id' => $account['id'],
+                'hashtag_id' => $tag->id
+            ],
+            [
+                'user_id' => $request->user()->id
+            ]
+        );
+
+        HashtagService::follow($pid, $tag->id);
+        HashtagFollowService::add($tag->id, $pid);
+
+        return response()->json(FollowedTagResource::make($follows)->toArray($request));
+    }
+
+    /**
+    * POST /api/v1/tags/:id/unfollow
+    *
+    *
+    * @return object
+    */
+    public function unfollowHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        $follows = HashtagFollow::whereProfileId($pid)
+            ->whereHashtagId($tag->id)
+            ->first();
+
+        if(!$follows) {
+            return [
+                'name' => $tag->name,
+                'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        if($follows) {
+            HashtagService::unfollow($pid, $tag->id);
+            HashtagFollowService::unfollow($tag->id, $pid);
+            HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
+            $follows->delete();
+        }
+
+        $res = FollowedTagResource::make($follows)->toArray($request);
+        $res['following'] = false;
+        return response()->json($res);
+    }
+
+    /**
+    * GET /api/v1/tags/:id
+    *
+    *
+    * @return object
+    */
+    public function getHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        if(!$tag) {
+            return [
+                'name' => $id,
+                'url' => config('app.url') . '/i/web/hashtag/' . $id,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        $res = [
+            'name' => $tag->name,
+            'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+            'history' => [],
+            'following' => HashtagService::isFollowing($pid, $tag->id)
+        ];
+
+        if($request->has(self::PF_API_ENTITY_KEY)) {
+            $res['count'] = HashtagService::count($tag->id);
+        }
+
+        return $this->json($res);
+    }
+
+    /**
+    * GET /api/v1/followed_tags
+    *
+    *
+    * @return array
+    */
+    public function getFollowedTags(Request $request)
+    {
+        abort_if(!$request->user(), 403);
+
+        $account = AccountService::get($request->user()->profile_id);
+
+        $this->validate($request, [
+            'cursor' => 'sometimes',
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+
+        $res = HashtagFollow::whereProfileId($account['id'])
+            ->orderByDesc('id')
+            ->cursorPaginate($limit)
+            ->withQueryString();
+
+        $pagination = false;
+        $prevPage = $res->nextPageUrl();
+        $nextPage = $res->previousPageUrl();
+        if($nextPage && $prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
+        } else if($nextPage && !$prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next"';
+        } else if(!$nextPage && $prevPage) {
+            $pagination = '<' . $prevPage . '>; rel="prev"';
+        }
+
+        if($pagination) {
+            return response()->json(FollowedTagResource::collection($res)->collection)
+                ->header('Link', $pagination);
+        }
+        return response()->json(FollowedTagResource::collection($res)->collection);
+    }
+}

app/Http/Controllers/CollectionController.php

@@ -153,7 +153,7 @@ class CollectionController extends Controller
             abort(400, 'You can only add '.$max.' posts per collection');
         }
 
-        $status = Status::whereScope('public')
+        $status = Status::whereIn('scope', ['public', 'unlisted'])
             ->whereProfileId($profileId)
             ->whereIn('type', ['photo', 'photo:album', 'video'])
             ->findOrFail($postId);
@@ -166,17 +166,13 @@ class CollectionController extends Controller
             'order'         => $count,
         ]);
 
-        CollectionService::addItem(
-        	$collection->id,
-        	$status->id,
-        	$count
-        );
+        CollectionService::deleteCollection($collection->id);
 
         $collection->updated_at = now();
         $collection->save();
         CollectionService::setCollection($collection->id, $collection);
 
-        return StatusService::get($status->id);
+        return StatusService::get($status->id, false);
     }
 
     public function getCollection(Request $request, $id)
@@ -226,10 +222,10 @@ class CollectionController extends Controller
 
         return collect($items)
         	->map(function($id) {
-        		return StatusService::get($id);
+                return StatusService::get($id, false);
         	})
         	->filter(function($item) {
-        		return $item && isset($item['account'], $item['media_attachments']);
+                return $item && ($item['visibility'] == 'public' ||  $item['visibility'] == 'unlisted') && isset($item['account'], $item['media_attachments']);
         	})
         	->values();
     }
@@ -298,7 +294,7 @@ class CollectionController extends Controller
             abort(400, 'You cannot delete the only post of a collection!');
         }
 
-        $status = Status::whereScope('public')
+        $status = Status::whereIn('scope', ['public', 'unlisted'])
             ->whereIn('type', ['photo', 'photo:album', 'video'])
             ->findOrFail($postId);
 

app/Http/Controllers/ComposeController.php

@@ -415,7 +415,7 @@ class ComposeController extends Controller
 		$results = Profile::select('id','domain','username')
 			->whereNotIn('id', $blocked)
 			->where('username','like','%'.$q.'%')
-			->groupBy('domain')
+			->groupBy('id', 'domain')
 			->limit(15)
 			->get()
 			->map(function($profile) {

app/Http/Controllers/DirectMessageController.php

@@ -17,12 +17,15 @@ use App\{
 use App\Services\MediaPathService;
 use App\Services\MediaBlocklistService;
 use App\Jobs\StatusPipeline\NewStatusPipeline;
+use App\Jobs\StatusPipeline\StatusDelete;
 use Illuminate\Support\Str;
 use App\Util\ActivityPub\Helpers;
 use App\Services\AccountService;
 use App\Services\StatusService;
 use App\Services\WebfingerService;
 use App\Models\Conversation;
+use App\Jobs\DirectPipeline\DirectDeletePipeline;
+use App\Jobs\DirectPipeline\DirectDeliverPipeline;
 
 class DirectMessageController extends Controller
 {
@@ -500,6 +503,8 @@ class DirectMessageController extends Controller
 		if($recipient['local'] == false) {
 			$dmc = $dm;
 			$this->remoteDelete($dmc);
+		} else {
+			StatusDelete::dispatch($status)->onQueue('high');
 		}
 
 		if(Conversation::whereStatusId($sid)->count()) {
@@ -541,9 +546,7 @@ class DirectMessageController extends Controller
 
 		StatusService::del($status->id, true);
 
-		$status->delete();
-		$dm->delete();
-
+		$status->forceDeleteQuietly();
 		return [200];
 	}
 
@@ -829,7 +832,7 @@ class DirectMessageController extends Controller
 			]
 		];
 
-		Helpers::sendSignedObject($profile, $url, $body);
+		DirectDeliverPipeline::dispatch($profile, $url, $body)->onQueue('high');
 	}
 
 	public function remoteDelete($dm)
@@ -852,7 +855,6 @@ class DirectMessageController extends Controller
 				'type' => 'Tombstone'
 			]
 		];
-
-		Helpers::sendSignedObject($profile, $url, $body);
+		DirectDeletePipeline::dispatch($profile, $url, $body)->onQueue('high');
 	}
 }

app/Http/Controllers/FederationController.php

@@ -30,6 +30,7 @@ use App\Util\ActivityPub\{
 };
 use Zttp\Zttp;
 use App\Services\InstanceService;
+use App\Services\AccountService;
 
 class FederationController extends Controller
 {
@@ -123,12 +124,15 @@ class FederationController extends Controller
             return redirect('/' . $username);
         }
 
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['statuses_count']), 404);
         $res = [
             '@context' => 'https://www.w3.org/ns/activitystreams',
             'id' => 'https://' . config('pixelfed.domain.app') . '/users/' . $username . '/outbox',
             'type' => 'OrderedCollection',
-			'totalItems' => 0,
-			'orderedItems' => []
+            'totalItems' => $account['statuses_count'] ?? 0,
         ];
 
         return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json');
@@ -239,12 +243,15 @@ class FederationController extends Controller
     {
         abort_if(!config_cache('federation.activitypub.enabled'), 404);
 
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['following_count']), 404);
         $obj = [
             '@context' => 'https://www.w3.org/ns/activitystreams',
             'id'       => $request->getUri(),
-			'type'     => 'OrderedCollectionPage',
-			'totalItems' => 0,
-			'orderedItems' => []
+            'type'     => 'OrderedCollection',
+            'totalItems' => $account['following_count'] ?? 0,
         ];
         return response()->json($obj);
     }
@@ -252,15 +259,16 @@ class FederationController extends Controller
     public function userFollowers(Request $request, $username)
     {
         abort_if(!config_cache('federation.activitypub.enabled'), 404);
-
+        $id = AccountService::usernameToId($username);
+        abort_if(!$id, 404);
+        $account = AccountService::get($id);
+        abort_if(!$account || !isset($account['followers_count']), 404);
         $obj = [
             '@context' => 'https://www.w3.org/ns/activitystreams',
             'id'       => $request->getUri(),
-			'type'     => 'OrderedCollectionPage',
-			'totalItems' => 0,
-			'orderedItems' => []
+            'type'     => 'OrderedCollection',
+            'totalItems' => $account['followers_count'] ?? 0,
         ];
-
         return response()->json($obj);
     }
 }

app/Http/Controllers/ImportPostController.php

@@ -83,6 +83,17 @@ class ImportPostController extends Controller
         );
     }
 
+    public function formatHashtags($val = false)
+    {
+        if(!$val || !strlen($val)) {
+            return null;
+        }
+
+        $groupedHashtagRegex = '/#\w+(?=#)/';
+
+        return preg_replace($groupedHashtagRegex, '$0 ', $val);
+    }
+
     public function store(Request $request)
     {
         abort_unless(config('import.instagram.enabled'), 404);
@@ -128,11 +139,11 @@ class ImportPostController extends Controller
             $ip->media = $c->map(function($m) {
                 return [
                     'uri' => $m['uri'],
-                    'title' => $m['title'],
+                    'title' => $this->formatHashtags($m['title']),
                     'creation_timestamp' => $m['creation_timestamp']
                 ];
             })->toArray();
-            $ip->caption = $c->count() > 1 ? $file['title'] : $ip->media[0]['title'];
+            $ip->caption = $c->count() > 1 ? $this->formatHashtags($file['title']) : $this->formatHashtags($ip->media[0]['title']);
             $ip->filename = last(explode('/', $ip->media[0]['uri']));
             $ip->metadata = $c->map(function($m) {
                 return [

app/Http/Controllers/LikeController.php

@@ -25,8 +25,7 @@ class LikeController extends Controller
 			'item'    => 'required|integer|min:1',
 		]);
 
-		// API deprecated
-		return;
+		abort(422, 'Deprecated API Endpoint');
 
 		$user = Auth::user();
 		$profile = $user->profile;
@@ -34,7 +33,7 @@ class LikeController extends Controller
 
 		if (Like::whereStatusId($status->id)->whereProfileId($profile->id)->exists()) {
 			$like = Like::whereProfileId($profile->id)->whereStatusId($status->id)->firstOrFail();
-			UnlikePipeline::dispatch($like);
+			UnlikePipeline::dispatch($like)->onQueue('feed');
 		} else {
 			abort_if(
 				Like::whereProfileId($user->profile_id)
@@ -60,7 +59,7 @@ class LikeController extends Controller
 					]) == false;
 				$like->save();
 				$status->save();
-				LikePipeline::dispatch($like);
+				LikePipeline::dispatch($like)->onQueue('feed');
 			}
 		}
 

app/Http/Controllers/RemoteAuthController.php

@@ -23,7 +23,13 @@ class RemoteAuthController extends Controller
 {
     public function start(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -37,7 +43,13 @@ class RemoteAuthController extends Controller
 
     public function getAuthDomains(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
 
         if(config('remote-auth.mastodon.domains.only_custom')) {
             $res = config('remote-auth.mastodon.domains.custom');
@@ -69,7 +81,14 @@ class RemoteAuthController extends Controller
 
     public function redirect(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $this->validate($request, ['domain' => 'required']);
 
         $domain = $request->input('domain');
@@ -158,6 +177,14 @@ class RemoteAuthController extends Controller
 
     public function preflight(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) {
             return redirect('/login');
         }
@@ -167,6 +194,14 @@ class RemoteAuthController extends Controller
 
     public function handleCallback(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $domain = $request->session()->get('oauth_domain');
 
         if($request->filled('code')) {
@@ -195,7 +230,13 @@ class RemoteAuthController extends Controller
 
     public function onboarding(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -204,6 +245,13 @@ class RemoteAuthController extends Controller
 
     public function sessionCheck(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -248,6 +296,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonData(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -279,6 +334,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateUsername(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -334,6 +396,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateEmail(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -359,6 +428,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonFollowers(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -386,6 +462,13 @@ class RemoteAuthController extends Controller
 
     public function handleSubmit(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -464,7 +547,13 @@ class RemoteAuthController extends Controller
 
     public function storeBio(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -483,7 +572,13 @@ class RemoteAuthController extends Controller
 
     public function accountToId(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -525,7 +620,13 @@ class RemoteAuthController extends Controller
 
     public function storeAvatar(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         $this->validate($request, [
             'avatar_url' => 'required|active_url',
@@ -547,7 +648,13 @@ class RemoteAuthController extends Controller
 
     public function finishUp(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
 
         $currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app');
@@ -564,7 +671,13 @@ class RemoteAuthController extends Controller
 
     public function handleLogin(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);

app/Http/Controllers/Settings/PrivacySettings.php

@@ -14,6 +14,7 @@ use App\Util\Lexer\PrettyNumber;
 use App\Util\ActivityPub\Helpers;
 use Auth, Cache, DB;
 use Illuminate\Http\Request;
+use App\Models\UserDomainBlock;
 
 trait PrivacySettings
 {
@@ -39,9 +40,11 @@ trait PrivacySettings
           'public_dm',
           'show_profile_follower_count',
           'show_profile_following_count',
+          'indexable',
           'show_atom',
         ];
 
+        $profile->indexable = $request->input('indexable') == 'on';
         $profile->is_suggestable = $request->input('is_suggestable') == 'on';
         $profile->save();
 
@@ -70,6 +73,8 @@ trait PrivacySettings
                 } else {
                     $settings->{$field} = false;
                 }
+            } elseif ($field == 'indexable') {
+
             } else {
                 if ($form == 'on') {
                     $settings->{$field} = true;
@@ -145,47 +150,25 @@ trait PrivacySettings
 
     public function blockedInstances()
     {
-        $pid = Auth::user()->profile->id;
-        $filters = UserFilter::whereUserId($pid)
-            ->whereFilterableType('App\Instance')
-            ->whereFilterType('block')
-            ->orderByDesc('id')
-            ->paginate(10);
-        return view('settings.privacy.blocked-instances', compact('filters'));
+        // deprecated
+        abort(404);
+    }
+
+    public function domainBlocks()
+    {
+        return view('settings.privacy.domain-blocks');
     }
 
     public function blockedInstanceStore(Request $request)
     {
-        $this->validate($request, [
-            'domain' => 'required|url|min:1|max:120'
-        ]);
-        $domain = $request->input('domain');
-        if(Helpers::validateUrl($domain) == false) {
-            return abort(400, 'Invalid domain');
-        }
-        $domain = parse_url($domain, PHP_URL_HOST);
-        $instance = Instance::firstOrCreate(['domain' => $domain]);
-        $filter = new UserFilter;
-        $filter->user_id = Auth::user()->profile->id;
-        $filter->filterable_id = $instance->id;
-        $filter->filterable_type = 'App\Instance';
-        $filter->filter_type = 'block';
-        $filter->save();
-        return response()->json(['msg' => 200]);
+        // deprecated
+        abort(404);
     }
 
     public function blockedInstanceUnblock(Request $request)
     {
-        $this->validate($request, [
-            'id'    => 'required|integer|min:1'
-        ]);
-        $pid = Auth::user()->profile->id;
-
-        $filter = UserFilter::whereFilterableType('App\Instance')
-            ->whereUserId($pid)
-            ->findOrFail($request->input('id'));
-        $filter->delete();
-        return redirect(route('settings.privacy.blocked-instances'));
+        // deprecated
+        abort(404);
     }
 
     public function blockedKeywords()

app/Http/Controllers/Stories/StoryApiV1Controller.php

@@ -20,27 +20,43 @@ use App\Jobs\StoryPipeline\StoryViewDeliver;
 use App\Services\AccountService;
 use App\Services\MediaPathService;
 use App\Services\StoryService;
+use App\Http\Resources\StoryView as StoryViewResource;
 
 class StoryApiV1Controller extends Controller
 {
+    const RECENT_KEY = 'pf:stories:recent-by-id:';
+    const RECENT_TTL = 300;
+
     public function carousel(Request $request)
     {
         abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
         $pid = $request->user()->profile_id;
 
         if(config('database.default') == 'pgsql') {
-			$s = Story::select('stories.*', 'followers.following_id')
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
                     ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
                     ->where('followers.profile_id', $pid)
                     ->where('stories.active', true)
-				->get();
+                    ->map(function($s) {
+                        $r  = new \StdClass;
+                        $r->id = $s->id;
+                        $r->profile_id = $s->profile_id;
+                        $r->type = $s->type;
+                        $r->path = $s->path;
+                        return $r;
+                    })
+                    ->unique('profile_id');
+            });
         } else {
-			$s = Story::select('stories.*', 'followers.following_id')
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
                     ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
                     ->where('followers.profile_id', $pid)
                     ->where('stories.active', true)
                     ->orderBy('id')
                     ->get();
+            });
         }
 
         $nodes = $s->map(function($s) use($pid) {
@@ -120,6 +136,115 @@ class StoryApiV1Controller extends Controller
         return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
     }
 
+    public function selfCarousel(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $pid = $request->user()->profile_id;
+
+        if(config('database.default') == 'pgsql') {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->map(function($s) {
+                        $r  = new \StdClass;
+                        $r->id = $s->id;
+                        $r->profile_id = $s->profile_id;
+                        $r->type = $s->type;
+                        $r->path = $s->path;
+                        return $r;
+                    })
+                    ->unique('profile_id');
+            });
+        } else {
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
+                return Story::select('stories.*', 'followers.following_id')
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('followers.profile_id', $pid)
+                    ->where('stories.active', true)
+                    ->orderBy('id')
+                    ->get();
+            });
+        }
+
+        $nodes = $s->map(function($s) use($pid) {
+            $profile = AccountService::get($s->profile_id, true);
+            if(!$profile || !isset($profile['id'])) {
+                return false;
+            }
+
+            return [
+                'id' => (string) $s->id,
+                'pid' => (string) $s->profile_id,
+                'type' => $s->type,
+                'src' => url(Storage::url($s->path)),
+                'duration' => $s->duration ?? 3,
+                'seen' => StoryService::hasSeen($pid, $s->id),
+                'created_at' => $s->created_at->format('c')
+            ];
+        })
+        ->filter()
+        ->groupBy('pid')
+        ->map(function($item) use($pid) {
+            $profile = AccountService::get($item[0]['pid'], true);
+            $url = $profile['local'] ? url("/stories/{$profile['username']}") :
+                url("/i/rs/{$profile['id']}");
+            return [
+                'id' => 'pfs:' . $profile['id'],
+                'user' => [
+                    'id' => (string) $profile['id'],
+                    'username' => $profile['username'],
+                    'username_acct' => $profile['acct'],
+                    'avatar' => $profile['avatar'],
+                    'local' => $profile['local'],
+                    'is_author' => $profile['id'] == $pid
+                ],
+                'nodes' => $item,
+                'url' => $url,
+                'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
+            ];
+        })
+        ->sortBy('seen')
+        ->values();
+
+        $selfProfile = AccountService::get($pid, true);
+        $res = [
+            'self' => [
+                'user' => [
+                    'id' => (string) $selfProfile['id'],
+                    'username' => $selfProfile['acct'],
+                    'avatar' => $selfProfile['avatar'],
+                    'local' => $selfProfile['local'],
+                    'is_author' => true
+                ],
+
+                'nodes' => [],
+            ],
+            'nodes' => $nodes,
+        ];
+
+        if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
+            $selfStories = Story::whereProfileId($pid)
+                ->whereActive(true)
+                ->get()
+                ->map(function($s) use($pid) {
+                    return [
+                        'id' => (string) $s->id,
+                        'type' => $s->type,
+                        'src' => url(Storage::url($s->path)),
+                        'duration' => $s->duration,
+                        'seen' => true,
+                        'created_at' => $s->created_at->format('c')
+                    ];
+                })
+                ->sortBy('id')
+                ->values();
+            $res['self']['nodes'] = $selfStories;
+        }
+        return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+    }
+
     public function add(Request $request)
     {
         abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
@@ -355,4 +480,26 @@ class StoryApiV1Controller extends Controller
         $path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
         return $path;
     }
+
+    public function viewers(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
+        $this->validate($request, [
+            'sid' => 'required|string|min:1|max:50'
+        ]);
+
+        $pid = $request->user()->profile_id;
+        $sid = $request->input('sid');
+
+        $story = Story::whereProfileId($pid)
+            ->whereActive(true)
+            ->findOrFail($sid);
+
+        $viewers = StoryView::whereStoryId($story->id)
+            ->orderByDesc('id')
+            ->cursorPaginate(10);
+
+        return StoryViewResource::collection($viewers);
+    }
 }

app/Http/Resources/StoryView.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Request;
+use Illuminate\Http\Resources\Json\JsonResource;
+use App\Services\AccountService;
+
+class StoryView extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @return array<string, mixed>
+     */
+    public function toArray(Request $request)
+    {
+        return AccountService::get($this->profile_id, true);
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+
+class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        AvatarService::cleanup($this->avatar, true);
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+use Illuminate\Support\Str;
+
+class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $avatar = $this->avatar;
+
+        $disk = AvatarService::disk();
+
+        $files = collect(AvatarService::storage($avatar));
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/CreateAvatar.php

@@ -2,19 +2,25 @@
 
 namespace App\Jobs\AvatarPipeline;
 
-use App\Avatar;
-use App\Profile;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Avatar;
+use App\Profile;
 
-class CreateAvatar implements ShouldQueue
+class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-    protected $profile;
+    public $profile;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
 
     /**
      * Delete the job if its models no longer exist.
@@ -23,6 +29,31 @@ class CreateAvatar implements ShouldQueue
      */
     public $deleteWhenMissingModels = true;
 
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:create:' . $this->profile->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
+    }
+    
     /**
      * Create a new job instance.
      *
@@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
      */
     public function __construct(Profile $profile)
     {
-        $this->profile = $profile;
+        $this->profile = $profile->withoutRelations();
     }
 
     /**
@@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
     public function handle()
     {
         $profile = $this->profile;
+        $isRemote = (bool) $profile->private_key == null;
         $path = 'public/avatars/default.jpg';
-        $avatar = new Avatar();
-        $avatar->profile_id = $profile->id;
-        $avatar->media_path = $path;
-        $avatar->change_count = 0;
-        $avatar->last_processed_at = \Carbon\Carbon::now();
-        $avatar->save();
+        Avatar::updateOrCreate(
+            [
+                'profile_id' => $profile->id,
+            ],
+            [
+                'media_path' => $path,
+                'change_count' => 0,
+                'is_remote' => $isRemote,
+                'last_processed_at' => now()
+            ]
+        );
     }
 }

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
+		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;
 	}

app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

@@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
 			$avatar->save();
 		}
 
-
 		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;

app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php

@@ -57,7 +57,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue
         $status = $this->status;
 
         if(AccountService::get($status->profile_id, true)) {
-            DecrementPostCount::dispatch($status->profile_id)->onQueue('feed');
+            DecrementPostCount::dispatch($status->profile_id)->onQueue('low');
         }
 
         NetworkTimelineService::del($status->id);

app/Jobs/DirectPipeline/DirectDeletePipeline.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Jobs\DirectPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Util\ActivityPub\Helpers;
+
+class DirectDeletePipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+
+    protected $profile;
+    protected $url;
+    protected $payload;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($profile, $url, $payload)
+    {
+        $this->profile = $profile;
+        $this->url = $url;
+        $this->payload = $payload;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
+    }
+}

app/Jobs/DirectPipeline/DirectDeliverPipeline.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Jobs\DirectPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Util\ActivityPub\Helpers;
+
+class DirectDeliverPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+
+    protected $profile;
+    protected $url;
+    protected $payload;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($profile, $url, $payload)
+    {
+        $this->profile = $profile;
+        $this->url = $url;
+        $this->payload = $payload;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
+    }
+}

app/Jobs/FollowPipeline/FollowServiceWarmCache.php

@@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use Cache;
 use DB;
+use Storage;
+use App\Follower;
 use App\Profile;
 
 class FollowServiceWarmCache implements ShouldQueue
@@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue
 	public $timeout = 5000;
 	public $failOnTimeout = false;
 
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->profileId))->dontRelease()];
+    }
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue
 	{
 		$id = $this->profileId;
 
+        if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) {
+            return;
+        }
+
 		$account = AccountService::get($id, true);
 
 		if(!$account) {
@@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue
 			return;
 		}
 
-		DB::table('followers')
-			->select('id', 'following_id', 'profile_id')
-			->whereFollowingId($id)
-			->orderBy('id')
-			->chunk(200, function($followers) use($id) {
-			foreach($followers as $follow) {
-				FollowerService::add($follow->profile_id, $id);
-			}
-		});
+        $hasFollowerPostProcessing = false;
+        $hasFollowingPostProcessing = false;
 
-		DB::table('followers')
-			->select('id', 'following_id', 'profile_id')
-			->whereProfileId($id)
-			->orderBy('id')
-			->chunk(200, function($followers) use($id) {
-			foreach($followers as $follow) {
-				FollowerService::add($id, $follow->following_id);
+        if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
+            $following = [];
+            $followers = [];
+    		foreach(Follower::where('following_id', $id)->orWhere('profile_id', $id)->lazyById(500) as $follow) {
+                if($follow->following_id != $id && $follow->profile_id != $id) {
+                    continue;
+                }
+                if($follow->profile_id == $id) {
+                    $following[] = $follow->following_id;
+                } else {
+                    $followers[] = $follow->profile_id;
+                }
+            }
+
+            if(count($followers) > 100) {
+                // store follower ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers));
+                $hasFollowerPostProcessing = true;
+            } else {
+                foreach($followers as $follower) {
+                    FollowerService::add($follower, $id);
+                }
+            }
+
+            if(count($following) > 100) {
+                // store following ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following));
+                $hasFollowingPostProcessing = true;
+            } else {
+                foreach($following as $following) {
+                    FollowerService::add($id, $following);
+                }
+            }
         }
-		});
 
 		Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800);
 		Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800);
@@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue
 
 		AccountService::del($id);
 
+        if($hasFollowingPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow');
+        }
+
+        if($hasFollowerPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow');
+        }
+
 		return;
 	}
 }

app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Jobs\FollowPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\AccountService;
+use App\Services\FollowerService;
+use Cache;
+use DB;
+use Storage;
+use App\Follower;
+use App\Profile;
+
+class FollowServiceWarmCacheLargeIngestPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $profileId;
+    public $followType;
+    public $tries = 5;
+    public $timeout = 5000;
+    public $failOnTimeout = false;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($profileId, $followType = 'following')
+    {
+        $this->profileId = $profileId;
+        $this->followType = $followType;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $pid = $this->profileId;
+        $type = $this->followType;
+
+        if($type === 'followers') {
+            $key = 'follow-warm-cache/' . $pid . '/followers.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($id, $pid, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        if($type === 'following') {
+            $key = 'follow-warm-cache/' . $pid . '/following.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($pid, $id, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        sleep(random_int(2, 5));
+        $files = Storage::files('follow-warm-cache/' . $pid);
+        if(empty($files)) {
+            Storage::deleteDirectory('follow-warm-cache/' . $pid);
+        }
+    }
+}

app/Jobs/FollowPipeline/UnfollowPipeline.php

@@ -17,6 +17,7 @@ use Illuminate\Support\Facades\Redis;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use App\Services\NotificationService;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UnfollowPipeline implements ShouldQueue
 {
@@ -55,6 +56,8 @@ class UnfollowPipeline implements ShouldQueue
 			return;
 		}
 
+		FeedUnfollowPipeline::dispatch($actor, $target)->onQueue('follow');
+
 		FollowerService::remove($actor, $target);
 
 		$actorProfileSync = Cache::get(FollowerService::FOLLOWING_SYNC_KEY . $actor);

app/Jobs/HomeFeedPipeline/FeedFollowPipeline.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\HomeTimelineService;
+use App\Services\SnowflakeService;
+use App\Status;
+
+class FeedFollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $minId = SnowflakeService::byDate(now()->subWeeks(6));
+
+        $ids = Status::where('id', '>', $minId)
+            ->where('profile_id', $followingId)
+            ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+            ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+            ->whereIn('visibility',['public', 'unlisted', 'private'])
+            ->orderByDesc('id')
+            ->limit(HomeTimelineService::FOLLOWER_FEED_POST_LIMIT)
+            ->pluck('id');
+
+        foreach($ids as $id) {
+            HomeTimelineService::add($actorId, $id);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        HomeTimelineService::add($this->pid, $this->sid);
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')
+            ->whereFilterableId($status['account']['id'])
+            ->whereIn('filter_type', ['mute', 'block'])
+            ->pluck('user_id')
+            ->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php

@@ -0,0 +1,112 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')
+            ->whereFilterableId($status['account']['id'])
+            ->whereIn('filter_type', ['mute', 'block'])
+            ->pluck('user_id')
+            ->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php

@@ -0,0 +1,98 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveDomainPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if(!config('exp.cached_home_timeline')) {
+            return;
+        }
+
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        if(!$this->pid || !$this->domain) {
+            return;
+        }
+        $domain = strtolower($this->domain);
+        $pid = $this->pid;
+        $posts = HomeTimelineService::get($pid, '0', '-1');
+
+        foreach($posts as $post) {
+            $status = StatusService::get($post, false);
+            if(!$status || !isset($status['url'])) {
+                HomeTimelineService::rem($pid, $post);
+                continue;
+            }
+            $host = strtolower(parse_url($status['url'], PHP_URL_HOST));
+            if($host === strtolower(config('pixelfed.domain.app')) || !$host) {
+                continue;
+            }
+            if($host === $domain) {
+                HomeTimelineService::rem($pid, $status['id']);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemovePipeline.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemovePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        HomeTimelineService::rem($this->pid, $this->sid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveRemotePipeline.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedUnfollowPipeline.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedUnfollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $ids = HomeTimelineService::get($actorId, 0, -1);
+        foreach($ids as $id) {
+            $status = StatusService::get($id, false);
+            if($status && isset($status['account'], $status['account']['id'])) {
+                if($status['account']['id'] == $followingId) {
+                    HomeTimelineService::rem($actorId, $id);
+                }
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedWarmCachePipeline.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\HomeTimelineService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class FeedWarmCachePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:warm-cache:pid:' . $this->pid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:warm-cache:pid:{$this->pid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid)
+    {
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $pid = $this->pid;
+        HomeTimelineService::warmCache($pid, true, 400, true);
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\UserFilter;
+use App\Models\UserDomainBlock;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $hashtag;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:insert:' . $this->hashtag->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:insert:{$this->hashtag->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(StatusHashtag $hashtag)
+    {
+        $this->hashtag = $hashtag;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hashtag = $this->hashtag;
+        $sid = $hashtag->status_id;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+        $skipIds = [];
+
+        if(strtolower(config('pixelfed.domain.app')) !== $domain) {
+            $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
+        }
+
+        $filters = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
+
+        if($filters && count($filters)) {
+            $skipIds = array_merge($skipIds, $filters);
+        }
+
+        $skipIds = array_unique(array_values($skipIds));
+
+        $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $hashtag->status_id);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php

@@ -0,0 +1,92 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $hid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:remove:' . $this->hid . ':' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:remove:{$this->hid}:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $hid)
+    {
+        $this->sid = $sid;
+        $this->hid = $hid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $hid = $this->hid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status || !isset($status['account']) || !isset($status['account']['id'])) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = HashtagFollowService::getPidByHid($hid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Support\Facades\Cache;
+use App\Follower;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class HashtagUnfollowPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $hid;
+    protected $slug;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($hid, $pid, $slug)
+    {
+        $this->hid = $hid;
+        $this->pid = $pid;
+        $this->slug = $slug;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hid = $this->hid;
+        $pid = $this->pid;
+        $slug = strtolower($this->slug);
+
+        $statusIds = HomeTimelineService::get($pid, 0, -1);
+
+        $followingIds = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
+            $following = Follower::whereProfileId($pid)->pluck('following_id');
+            return $following->push($pid)->toArray();
+        });
+
+        foreach($statusIds as $id) {
+            $status = StatusService::get($id, false);
+            if(!$status || empty($status['tags'])) {
+                HomeTimelineService::rem($pid, $id);
+                continue;
+            }
+            $following = in_array((int) $status['account']['id'], $followingIds);
+            if($following === true) {
+                continue;
+            }
+
+            $tags = collect($status['tags'])->map(function($tag) {
+                return strtolower($tag['name']);
+            })->filter()->values()->toArray();
+
+            if(in_array($slug, $tags)) {
+                HomeTimelineService::rem($pid, $id);
+            }
+        }
+    }
+}

app/Jobs/ImageOptimizePipeline/ImageResize.php

@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Log;
 
 class ImageResize implements ShouldQueue
 {
@@ -46,6 +47,7 @@ class ImageResize implements ShouldQueue
         }
         $path = storage_path('app/'.$media->media_path);
         if (!is_file($path) || $media->skip_optimize) {
+            Log::info('Tried to optimize media that does not exist or is not readable. ' . $path);
             return;
         }
 
@@ -57,6 +59,7 @@ class ImageResize implements ShouldQueue
             $img = new Image();
             $img->resizeImage($media);
         } catch (Exception $e) {
+            Log::error($e);
         }
 
         ImageThumbnail::dispatch($media)->onQueue('mmo');

app/Jobs/InboxPipeline/InboxValidator.php

@@ -193,7 +193,7 @@ class InboxValidator implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InboxPipeline/InboxWorker.php

@@ -173,7 +173,7 @@ class InboxWorker implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Jobs\InternalPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Notification;
+use Cache;
+use App\Services\NotificationService;
+
+class NotificationEpochUpdatePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 1500;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'ip:notification-epoch-update';
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping('ip:notification-epoch-update'))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $rec = Notification::where('created_at', '>', now()->subMonths(6))->first();
+        $id = 1;
+        if($rec) {
+            $id = $rec->id;
+        }
+        Cache::put(NotificationService::EPOCH_CACHE_KEY . '6', $id, 1209600);
+    }
+}

app/Jobs/MediaPipeline/MediaDeletePipeline.php

@@ -10,8 +10,11 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
+use App\Services\Media\MediaHlsService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class MediaDeletePipeline implements ShouldQueue
+class MediaDeletePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -20,8 +23,34 @@ class MediaDeletePipeline implements ShouldQueue
     public $timeout = 300;
     public $tries = 3;
     public $maxExceptions = 1;
+    public $failOnTimeout = true;
     public $deleteWhenMissingModels = true;
 
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:purge-job:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:purge-job:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
 	public function __construct(Media $media)
 	{
 		$this->media = $media;
@@ -63,9 +92,17 @@ class MediaDeletePipeline implements ShouldQueue
 			$disk->delete($thumb);
 		}
 
+		if($media->hls_path != null) {
+            $files = MediaHlsService::allFiles($media);
+            if($files && count($files)) {
+                foreach($files as $file) {
+                    $disk->delete($file);
+                }
+            }
+		}
+
 		$media->delete();
 
 		return 1;
 	}
-
 }

app/Jobs/ProfilePipeline/DecrementPostCount.php

@@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue
             return 1;
         }
 
-        if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-            $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-            $profile->save();
-            AccountService::del($id);
-        } else {
         $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0;
         $profile->save();
         AccountService::del($id);
-        }
 
         return 1;
     }

app/Jobs/ProfilePipeline/IncrementPostCount.php

@@ -8,16 +8,48 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use App\Profile;
 use App\Status;
 use App\Services\AccountService;
 
-class IncrementPostCount implements ShouldQueue
+class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
 	public $id;
 
+	public $timeout = 900;
+	public $tries = 3;
+	public $maxExceptions = 1;
+	public $failOnTimeout = true;
+
+	/**
+	 * The number of seconds after which the job's unique lock will be released.
+	 *
+	 * @var int
+	 */
+	public $uniqueFor = 3600;
+
+	/**
+	 * Get the unique ID for the job.
+	 */
+	public function uniqueId(): string
+	{
+		return 'propipe:ipc:' . $this->id;
+	}
+
+	/**
+	 * Get the middleware the job should pass through.
+	 *
+	 * @return array<int, object>
+	 */
+	public function middleware(): array
+	{
+		return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()];
+	}
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -43,17 +75,11 @@ class IncrementPostCount implements ShouldQueue
 			return 1;
 		}
 
-		if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-			$profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-			$profile->last_status_at = now();
-			$profile->save();
-			AccountService::del($id);
-		} else {
 		$profile->status_count = $profile->status_count + 1;
 		$profile->last_status_at = now();
 		$profile->save();
 		AccountService::del($id);
-		}
+		AccountService::get($id);
 
 		return 1;
 	}

app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php

@@ -0,0 +1,119 @@
+<?php
+
+namespace App\Jobs\ProfilePipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Follower;
+use App\Profile;
+use App\Notification;
+use DB;
+use App\Services\AccountService;
+use App\Services\FollowerService;
+use App\Services\NotificationService;
+
+class ProfilePurgeFollowersByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'followers:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("followers:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        $pid = $this->pid;
+        $domain = $this->domain;
+
+        $query = 'SELECT f.*
+            FROM followers f
+            JOIN profiles p ON p.id = f.profile_id OR p.id = f.following_id
+            WHERE (f.profile_id = ? OR f.following_id = ?)
+            AND p.domain = ?;';
+        $params = [$pid, $pid, $domain];
+
+        foreach(DB::cursor($query, $params) as $n) {
+            if(!$n || !$n->id) {
+                continue;
+            }
+            $follower = Follower::find($n->id);
+            if($follower->following_id == $pid && $follower->profile_id) {
+                FollowerService::remove($follower->profile_id, $pid, true);
+                $follower->delete();
+            } else if ($follower->profile_id == $pid && $follower->following_id) {
+                FollowerService::remove($follower->following_id, $pid, true);
+                $follower->delete();
+            }
+        }
+
+        $profile = Profile::find($pid);
+
+        $followerCount = DB::table('profiles')
+            ->join('followers', 'profiles.id', '=', 'followers.following_id')
+            ->where('followers.following_id', $pid)
+            ->count();
+
+        $followingCount = DB::table('profiles')
+            ->join('followers', 'profiles.id', '=', 'followers.following_id')
+            ->where('followers.profile_id', $pid)
+            ->count();
+
+        $profile->followers_count = $followerCount;
+        $profile->following_count = $followingCount;
+        $profile->save();
+
+        AccountService::del($profile->id);
+    }
+}

app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Jobs\ProfilePipeline;
+
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Notification;
+use DB;
+use App\Services\NotificationService;
+
+class ProfilePurgeNotificationsByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $domain;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'notify:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("notify:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid, $domain)
+    {
+        $this->pid = $pid;
+        $this->domain = $domain;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if ($this->batch()->cancelled()) {
+            return;
+        }
+
+        $pid = $this->pid;
+        $domain = $this->domain;
+
+        $query = 'SELECT notifications.*
+            FROM profiles
+            JOIN notifications on profiles.id = notifications.actor_id
+            WHERE notifications.profile_id = ?
+            AND profiles.domain = ?';
+        $params = [$pid, $domain];
+
+        foreach(DB::cursor($query, $params) as $n) {
+            if(!$n || !$n->id) {
+                continue;
+            }
+            Notification::where('id', $n->id)->delete();
+            NotificationService::del($pid, $n->id);
+        }
+    }
+}

app/Jobs/SharePipeline/SharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
 
 class SharePipeline implements ShouldQueue
 {
@@ -82,6 +83,8 @@ class SharePipeline implements ShouldQueue
 			]
 		);
 
+		FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		return $this->remoteAnnounceDeliver();
 	}
 

app/Jobs/SharePipeline/UndoSharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
 
 class UndoSharePipeline implements ShouldQueue
 {
@@ -35,6 +36,8 @@ class UndoSharePipeline implements ShouldQueue
 		$actor = $status->profile;
 		$parent = Status::find($status->reblog_of_id);
 
+		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		if($parent) {
 			$target = $parent->profile_id;
 			ReblogService::removePostReblog($parent->profile_id, $status->id);

app/Jobs/StatusPipeline/RemoteStatusDelete.php

@@ -21,9 +21,11 @@ use App\{
 };
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use League\Fractal;
 use Illuminate\Support\Str;
 use League\Fractal\Serializer\ArraySerializer;
@@ -37,8 +39,10 @@ use App\Services\AccountService;
 use App\Services\CollectionService;
 use App\Services\StatusService;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
+use App\Jobs\ProfilePipeline\DecrementPostCount;
+use App\Services\NotificationService;
 
-class RemoteStatusDelete implements ShouldQueue
+class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -51,9 +55,35 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public $deleteWhenMissingModels = true;
 
-    public $timeout = 90;
-    public $tries = 2;
-    public $maxExceptions = 1;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 180;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'status:remote:delete:' . $this->status->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()];
+    }
 
     /**
      * Create a new job instance.
@@ -62,7 +92,7 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public function __construct(Status $status)
     {
-        $this->status = $status;
+        $this->status = $status->withoutRelations();
     }
 
     /**
@@ -77,14 +107,10 @@ class RemoteStatusDelete implements ShouldQueue
         if($status->deleted_at) {
             return;
         }
-        $profile = $this->status->profile;
 
         StatusService::del($status->id, true);
 
-        if($profile->status_count && $profile->status_count > 0) {
-            $profile->status_count = $profile->status_count - 1;
-            $profile->save();
-        }
+        DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox');
 
         return $this->unlinkRemoveMedia($status);
     }
@@ -112,14 +138,34 @@ class RemoteStatusDelete implements ShouldQueue
                 CollectionService::removeItem($col->collection_id, $col->object_id);
                 $col->delete();
         });
-        DirectMessage::whereStatusId($status->id)->delete();
+        $dms = DirectMessage::whereStatusId($status->id)->get();
+        foreach($dms as $dm) {
+            $not = Notification::whereItemType('App\DirectMessage')
+                ->whereItemId($dm->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $dm->delete();
+        }
         Like::whereStatusId($status->id)->forceDelete();
         Media::whereStatusId($status->id)
         ->get()
         ->each(function($media) {
             MediaDeletePipeline::dispatch($media)->onQueue('mmo');
         });
-        MediaTag::where('status_id', $status->id)->delete();
+        $mediaTags = MediaTag::where('status_id', $status->id)->get();
+        foreach($mediaTags as $mtag) {
+            $not = Notification::whereItemType('App\MediaTag')
+                ->whereItemId($mtag->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $mtag->delete();
+        }
         Mention::whereStatusId($status->id)->forceDelete();
         Notification::whereItemType('App\Status')
             ->whereItemId($status->id)

app/Jobs/StatusPipeline/StatusDelete.php

@@ -35,6 +35,7 @@ use GuzzleHttp\Promise;
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\CollectionService;
 use App\Services\StatusService;
+use App\Services\NotificationService;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 
 class StatusDelete implements ShouldQueue
@@ -115,10 +116,30 @@ class StatusDelete implements ShouldQueue
                 $col->delete();
         });
 
-        DirectMessage::whereStatusId($status->id)->delete();
+        $dms = DirectMessage::whereStatusId($status->id)->get();
+        foreach($dms as $dm) {
+            $not = Notification::whereItemType('App\DirectMessage')
+                ->whereItemId($dm->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $dm->delete();
+        }
         Like::whereStatusId($status->id)->delete();
 
-		MediaTag::where('status_id', $status->id)->delete();
+        $mediaTags = MediaTag::where('status_id', $status->id)->get();
+        foreach($mediaTags as $mtag) {
+            $not = Notification::whereItemType('App\MediaTag')
+                ->whereItemId($mtag->id)
+                ->first();
+            if($not) {
+                NotificationService::del($not->profile_id, $not->id);
+                $not->forceDeleteQuietly();
+            }
+            $mtag->delete();
+        }
         Mention::whereStatusId($status->id)->forceDelete();
 
 		Notification::whereItemType('App\Status')

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -19,7 +19,11 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use App\Services\StatusService;
 use App\Services\UserFilterService;
+use App\Services\AdminShadowFilterService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
 
 class StatusEntityLexer implements ShouldQueue
 {
@@ -104,12 +108,12 @@ class StatusEntityLexer implements ShouldQueue
             }
             DB::transaction(function () use ($status, $tag) {
                 $slug = str_slug($tag, '-', false);
-				$hashtag = Hashtag::where('slug', $slug)->first();
-				if (!$hashtag) {
-					$hashtag = Hashtag::create(
-						['name' => $tag, 'slug' => $slug]
-					);
-				}
+
+                $hashtag = Hashtag::firstOrCreate([
+                    'slug' => $slug
+                ], [
+                    'name' => $tag
+                ]);
 
                 StatusHashtag::firstOrCreate(
                     [
@@ -149,6 +153,21 @@ class StatusEntityLexer implements ShouldQueue
                 MentionPipeline::dispatch($status, $m);
             });
         }
+        $this->fanout();
+    }
+
+    public function fanout()
+    {
+        $status = $this->status;
+        StatusService::refresh($status->id);
+
+        if(config('exp.cached_home_timeline')) {
+            if( $status->in_reply_to_id === null &&
+                in_array($status->scope, ['public', 'unlisted', 'private'])
+            ) {
+                FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+            }
+        }
         $this->deliver();
     }
 
@@ -176,8 +195,10 @@ class StatusEntityLexer implements ShouldQueue
             $status->reblog_of_id === null &&
             ($hideNsfw ? $status->is_nsfw == false : true)
         ) {
+            if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
                 PublicTimelineService::add($status->id);
             }
+        }
 
         if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
             StatusActivityPubDeliver::dispatch($status);

app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php

@@ -90,7 +90,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
 			]);
 
 		$nm->each(function($n, $key) use($status) {
-			$res = Http::retry(3, 100, throw: false)->head($n['url']);
+			$res = Http::withOptions(['allow_redirects' => false])->retry(3, 100, throw: false)->head($n['url']);
 
 			if(!$res->successful()) {
 				return;

app/Jobs/StatusPipeline/StatusTagsPipeline.php

@@ -45,6 +45,11 @@ class StatusTagsPipeline implements ShouldQueue
     {
         $res = $this->activity;
         $status = $this->status;
+
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
+
         $tags = collect($res['tag']);
 
         // Emoji
@@ -73,19 +78,18 @@ class StatusTagsPipeline implements ShouldQueue
 
             if(config('database.default') === 'pgsql') {
                 $hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name))
+                    ->orWhere('slug', 'ilike', str_slug($name, '-', false))
                     ->first();
 
                 if(!$hashtag) {
-            		$hashtag = new Hashtag;
-            		$hashtag->name = $name;
-            		$hashtag->slug = str_slug($name);
-            		$hashtag->save();
+                    $hashtag = Hashtag::updateOrCreate([
+                        'slug' => str_slug($name, '-', false),
+                        'name' => $name
+                    ]);
                 }
             } else {
-				$hashtag = Hashtag::firstOrCreate([
-					'slug' => str_slug($name)
-				], [
+                $hashtag = Hashtag::updateOrCreate([
+                    'slug' => str_slug($name, '-', false),
                     'name' => $name
                 ]);
             }
@@ -128,5 +132,7 @@ class StatusTagsPipeline implements ShouldQueue
             $mention->save();
             MentionPipeline::dispatch($status, $mention);
         });
+
+        StatusService::refresh($status->id);
     }
 }

app/Jobs/VideoPipeline/VideoHlsPipeline.php

@@ -0,0 +1,109 @@
+<?php
+
+namespace App\Jobs\VideoPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use FFMpeg\Format\Video\X264;
+use FFMpeg;
+use Cache;
+use App\Services\MediaService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class VideoHlsPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $media;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-hls:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-hls:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($media)
+    {
+        $this->media = $media;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $depCheck = Cache::rememberForever('video-pipeline:hls:depcheck', function() {
+            $bin = config('laravel-ffmpeg.ffmpeg.binaries');
+            $output = shell_exec($bin . ' -version');
+            if($output && preg_match('/ffmpeg version ([^\s]+)/', $output, $matches)) {
+                $version = $matches[1];
+                return (version_compare($version, config('laravel-ffmpeg.min_hls_version')) >= 0) ? 'ok' : false;
+            } else {
+                return false;
+            }
+        });
+
+        if(!$depCheck || $depCheck !== 'ok') {
+            return;
+        }
+
+        $media = $this->media;
+
+        $bitrate = (new X264)->setKiloBitrate(config('media.hls.bitrate') ?? 1000);
+
+        $mp4 = $media->media_path;
+        $man = str_replace('.mp4', '.m3u8', $mp4);
+
+        FFMpeg::fromDisk('local')
+            ->open($mp4)
+            ->exportForHLS()
+            ->setSegmentLength(16)
+            ->setKeyFrameInterval(48)
+            ->addFormat($bitrate)
+            ->save($man);
+
+        $media->hls_path = $man;
+        $media->hls_transcoded_at = now();
+        $media->save();
+
+        MediaService::del($media->status_id);
+        usleep(50000);
+        StatusService::del($media->status_id);
+
+        return;
+    }
+}

app/Jobs/VideoPipeline/VideoThumbnail.php

@@ -16,13 +16,46 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Util\Media\Blurhash;
 use App\Services\MediaService;
 use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class VideoThumbnail implements ShouldQueue
+class VideoThumbnail implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     protected $media;
 
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-thumb:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-thumb:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
     /**
      * Create a new job instance.
      *
@@ -54,7 +87,7 @@ class VideoThumbnail implements ShouldQueue
             $path[$i] = $t;
             $save = implode('/', $path);
             $video = FFMpeg::open($base)
-            ->getFrameFromSeconds(0)
+            ->getFrameFromSeconds(1)
             ->export()
             ->toDisk('local')
             ->save($save);
@@ -68,6 +101,9 @@ class VideoThumbnail implements ShouldQueue
                 $media->save();
             }
 
+            if(config('media.hls.enabled')) {
+                VideoHlsPipeline::dispatch($media)->onQueue('mmo');
+            }
         } catch (Exception $e) {
             
         }

app/Models/AdminShadowFilter.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use App\Services\AccountService;
+use App\Profile;
+
+class AdminShadowFilter extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    protected $casts = [
+        'created_at' => 'datetime'
+    ];
+
+    public function account()
+    {
+        if($this->item_type === 'App\Profile') {
+            return AccountService::get($this->item_id, true);
+        }
+
+        return;
+    }
+
+    public function profile()
+    {
+        return $this->belongsTo(Profile::class, 'item_id');
+    }
+}

app/Models/DefaultDomainBlock.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class DefaultDomainBlock extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+}

app/Models/HashtagRelated.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HashtagRelated extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    /**
+     * The attributes that should be mutated to dates and other custom formats.
+     *
+     * @var array
+     */
+    protected $casts = [
+        'related_tags' => 'array',
+        'last_calculated_at' => 'datetime',
+        'last_moderated_at' => 'datetime',
+    ];
+}

app/Models/UserDomainBlock.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use App\Profile;
+
+class UserDomainBlock extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    public $timestamps = false;
+
+    public function profile()
+    {
+        return $this->belongsTo(Profile::class, 'profile_id');
+    }
+}

app/Observers/FollowerObserver.php

@@ -5,6 +5,8 @@ namespace App\Observers;
 use App\Follower;
 use App\Services\FollowerService;
 use Cache;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class FollowerObserver
 {
@@ -21,6 +23,7 @@ class FollowerObserver
         }
 
         FollowerService::add($follower->profile_id, $follower->following_id);
+        FeedFollowPipeline::dispatch($follower->profile_id, $follower->following_id)->onQueue('follow');
     }
 
     /**

app/Observers/HashtagFollowObserver.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Observers;
+
+use App\HashtagFollow;
+use App\Services\HashtagFollowService;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
+
+class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
+{
+    /**
+     * Handle the HashtagFollow "created" event.
+     */
+    public function created(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::add($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "updated" event.
+     */
+    public function updated(HashtagFollow $hashtagFollow): void
+    {
+    	//
+    }
+
+    /**
+     * Handle the HashtagFollow "deleting" event.
+     */
+    public function deleting(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "restored" event.
+     */
+    public function restored(HashtagFollow $hashtagFollow): void
+    {
+        //
+    }
+
+    /**
+     * Handle the HashtagFollow "force deleted" event.
+     */
+    public function forceDeleted(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+}

app/Observers/StatusHashtagObserver.php

@@ -5,32 +5,31 @@ namespace App\Observers;
 use DB;
 use App\StatusHashtag;
 use App\Services\StatusHashtagService;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagRemoveFanoutPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
 
-class StatusHashtagObserver
+class StatusHashtagObserver implements ShouldHandleEventsAfterCommit
 {
-    /**
-     * Handle events after all transactions are committed.
-     *
-     * @var bool
-     */
-    public $afterCommit = true;
-
     /**
      * Handle the notification "created" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function created(StatusHashtag $hashtag)
     {
         StatusHashtagService::set($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->increment('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagInsertFanoutPipeline::dispatch($hashtag)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "updated" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function updated(StatusHashtag $hashtag)
@@ -41,19 +40,22 @@ class StatusHashtagObserver
     /**
      * Handle the notification "deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function deleted(StatusHashtag $hashtag)
     {
         StatusHashtagService::del($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->decrement('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagRemoveFanoutPipeline::dispatch($hashtag->status_id, $hashtag->hashtag_id)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "restored" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function restored(StatusHashtag $hashtag)
@@ -64,7 +66,7 @@ class StatusHashtagObserver
     /**
      * Handle the notification "force deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function forceDeleted(StatusHashtag $hashtag)

app/Observers/StatusObserver.php

@@ -7,6 +7,8 @@ use App\Services\ProfileStatusService;
 use Cache;
 use App\Models\ImportPost;
 use App\Services\ImportService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class StatusObserver
 {
@@ -63,6 +65,14 @@ class StatusObserver
             ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete();
             ImportService::clearImportedFiles($status->profile_id);
         }
+
+        if(config('exp.cached_home_timeline')) {
+        	if($status->uri) {
+        		FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	} else {
+        		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	}
+        }
     }
 
     /**

app/Observers/UserFilterObserver.php

@@ -4,6 +4,8 @@ namespace App\Observers;
 
 use App\UserFilter;
 use App\Services\UserFilterService;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UserFilterObserver
 {
@@ -78,10 +80,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::mute($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::block($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}
@@ -96,10 +100,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::unmute($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::unblock($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}

app/Observers/UserObserver.php

@@ -7,6 +7,9 @@ use App\Follower;
 use App\Profile;
 use App\User;
 use App\UserSetting;
+use App\Services\UserFilterService;
+use App\Models\DefaultDomainBlock;
+use App\Models\UserDomainBlock;
 use App\Jobs\FollowPipeline\FollowPipeline;
 use DB;
 use App\Services\FollowerService;
@@ -14,7 +17,18 @@ use App\Services\FollowerService;
 class UserObserver
 {
     /**
-	 * Listen to the User created event.
+     * Handle the notification "created" event.
+     *
+     * @param  \App\User $user
+     * @return void
+     */
+    public function created(User $user): void
+    {
+        $this->handleUser($user);
+    }
+
+    /**
+     * Listen to the User saved event.
      *
      * @param \App\User $user
      *
@@ -22,7 +36,38 @@ class UserObserver
      */
     public function saved(User $user)
     {
-		if($user->status == 'deleted') {
+        $this->handleUser($user);
+    }
+
+    /**
+     * Listen to the User updated event.
+     *
+     * @param \App\User $user
+     *
+     * @return void
+     */
+    public function updated(User $user): void
+    {
+        $this->handleUser($user);
+        if($user->profile) {
+            $this->applyDefaultDomainBlocks($user);
+        }
+    }
+
+    /**
+     * Handle the user "deleted" event.
+     *
+     * @param  \App\User $user
+     * @return void
+     */
+    public function deleted(User $user)
+    {
+        FollowerService::delCache($user->profile_id);
+    }
+
+    protected function handleUser($user)
+    {
+        if(in_array($user->status, ['deleted', 'delete'])) {
             return;
         }
 
@@ -49,9 +94,11 @@ class UserObserver
                 $profile->private_key = $pki_private;
                 $profile->public_key = $pki_public;
                 $profile->save();
+                $this->applyDefaultDomainBlocks($user);
                 return $profile;
             });
 
+
             DB::transaction(function() use($user, $profile) {
                 $user = User::findOrFail($user->id);
                 $user->profile_id = $profile->id;
@@ -92,14 +139,22 @@ class UserObserver
         }
     }
 
-    /**
-     * Handle the user "deleted" event.
-     *
-     * @param  \App\User $user
-     * @return void
-     */
-    public function deleted(User $user)
+    protected function applyDefaultDomainBlocks($user)
     {
-        FollowerService::delCache($user->profile_id);
+        if($user->profile_id == null) {
+            return;
+        }
+        $defaultDomainBlocks = DefaultDomainBlock::pluck('domain')->toArray();
+
+        if(!$defaultDomainBlocks || !count($defaultDomainBlocks)) {
+            return;
+        }
+
+        foreach($defaultDomainBlocks as $domain) {
+            UserDomainBlock::updateOrCreate([
+                'profile_id' => $user->profile_id,
+                'domain' => strtolower(trim($domain))
+            ]);
+        }
     }
 }

app/Services/AccountService.php

@@ -7,6 +7,7 @@ use App\Profile;
 use App\Status;
 use App\User;
 use App\UserSetting;
+use App\Models\UserDomainBlock;
 use App\Transformer\Api\AccountTransformer;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
@@ -166,7 +167,7 @@ class AccountService
     public static function usernameToId($username)
     {
         $key = self::CACHE_KEY . 'u2id:' . hash('sha256', $username);
-		return Cache::remember($key, 900, function() use($username) {
+        return Cache::remember($key, 14400, function() use($username) {
             $s = Str::of($username);
             if($s->contains('@') && !$s->startsWith('@')) {
                 $username = "@{$username}";
@@ -220,4 +221,27 @@ class AccountService
             return false;
         });
     }
+
+    public static function setLastActive($id = false)
+    {
+        if(!$id) { return; }
+        $key = 'user:last_active_at:id:' . $id;
+        if(!Cache::has($key)) {
+            $user = User::find($id);
+            if(!$user) { return; }
+            $user->last_active_at = now();
+            $user->save();
+            Cache::put($key, 1, 14400);
+        }
+        return;
+    }
+
+    public static function blocksDomain($pid, $domain = false)
+    {
+        if(!$domain) {
+            return;
+        }
+
+        return UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->exists();
+    }
 }

app/Services/ActivityPubFetchService.php

@@ -28,7 +28,7 @@ class ActivityPubFetchService
 		$headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
 
 		try {
-			$res = Http::withHeaders($headers)
+			$res = Http::withOptions(['allow_redirects' => false])->withHeaders($headers)
 				->timeout(30)
 				->connectTimeout(5)
 				->retry(3, 500)

app/Services/AdminShadowFilterService.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\AdminShadowFilter;
+use Cache;
+
+class AdminShadowFilterService
+{
+    const CACHE_KEY = 'pf:services:asfs:';
+
+    public static function queryFilter($name = 'hide_from_public_feeds')
+    {
+        return AdminShadowFilter::whereItemType('App\Profile')
+            ->whereActive(1)
+            ->where('hide_from_public_feeds', true)
+            ->pluck('item_id')
+            ->toArray();
+    }
+
+    public static function getHideFromPublicFeedsList($refresh = false)
+    {
+        $key = self::CACHE_KEY . 'list:hide_from_public_feeds';
+        if($refresh) {
+            Cache::forget($key);
+        }
+        return Cache::remember($key, 86400, function() {
+            return AdminShadowFilter::whereItemType('App\Profile')
+                ->whereActive(1)
+                ->where('hide_from_public_feeds', true)
+                ->pluck('item_id')
+                ->toArray();
+        });
+    }
+
+    public static function canAddToPublicFeedByProfileId($profileId)
+    {
+        return !in_array($profileId, self::getHideFromPublicFeedsList());
+    }
+
+    public static function refresh()
+    {
+        $keys = [
+            self::CACHE_KEY . 'list:hide_from_public_feeds'
+        ];
+
+        foreach($keys as $key) {
+            Cache::forget($key);
+        }
+    }
+}

app/Services/AvatarService.php

@@ -3,7 +3,13 @@
 namespace App\Services;
 
 use Cache;
+use Storage;
+use Illuminate\Support\Str;
+use App\Avatar;
 use App\Profile;
+use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
+use League\Flysystem\UnableToCheckDirectoryExistence;
+use League\Flysystem\UnableToRetrieveMetadata;
 
 class AvatarService
 {
@@ -20,4 +26,102 @@ class AvatarService
         }
         return $profile->avatarUrl();
     }
+
+    public static function disk()
+    {
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return false;
+        }
+
+        $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
+        $disk = Storage::disk($driver);
+
+        return $disk;
+    }
+
+    public static function storage(Avatar $avatar)
+    {
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        return $disk->allFiles($base . $avatar->profile_id);
+    }
+
+    public static function cleanup($avatar, $confirm = false)
+    {
+        if(!$avatar || !$confirm) {
+            return;
+        }
+
+        if($avatar->cdn_url == null) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return;
+        }
+
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        try {
+            $exists = $disk->directoryExists($base . $avatar->profile_id);
+        } catch (
+            UnableToRetrieveMetadata |
+            UnableToCheckDirectoryExistence |
+            Exception $e
+        ) {
+            return;
+        }
+
+        if(!$exists) {
+            return;
+        }
+
+        $files = collect($disk->allFiles($base . $avatar->profile_id));
+
+        if(!$files || !$files->count() || $files->count() === 1) {
+            return;
+        }
+
+        if($files->count() > 5) {
+            AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
+            return;
+        }
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
 }

app/Services/FollowerService.php

@@ -19,26 +19,34 @@ class FollowerService
     const FOLLOWING_SYNC_KEY = 'pf:services:followers:sync-following:';
     const FOLLOWING_KEY = 'pf:services:follow:following:id:';
     const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
+    const FOLLOWERS_LOCAL_KEY = 'pf:services:follow:local-follower-ids:';
+    const FOLLOWERS_INTER_KEY = 'pf:services:follow:followers:inter:id:';
 
-	public static function add($actor, $target)
+    public static function add($actor, $target, $refresh = true)
     {
         $ts = (int) microtime(true);
+        if($refresh) {
           RelationshipService::refresh($actor, $target);
+        } else {
+          RelationshipService::forget($actor, $target);
+        }
         Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target);
         Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor);
         Cache::forget('profile:following:' . $actor);
     }
 
-	public static function remove($actor, $target)
+    public static function remove($actor, $target, $silent = false)
     {
         Redis::zrem(self::FOLLOWING_KEY . $actor, $target);
         Redis::zrem(self::FOLLOWERS_KEY . $target, $actor);
-		Cache::forget('pf:services:follower:audience:' . $actor);
-		Cache::forget('pf:services:follower:audience:' . $target);
+        if($silent !== true) {
             AccountService::del($actor);
             AccountService::del($target);
             RelationshipService::refresh($actor, $target);
             Cache::forget('profile:following:' . $actor);
+        } else {
+            RelationshipService::forget($actor, $target);
+        }
     }
 
     public static function followers($id, $start = 0, $stop = 10)
@@ -83,12 +91,16 @@ class FollowerService
         return Redis::zCard(self::FOLLOWING_KEY . $id);
     }
 
-	public static function follows(string $actor, string $target)
+    public static function follows(string $actor, string $target, $quickCheck = false)
     {
         if($actor == $target) {
             return false;
         }
 
+        if($quickCheck) {
+            return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
+        }
+
         if(self::followerCount($target, false) && self::followingCount($actor, false)) {
             self::cacheSyncCheck($target, 'followers');
             return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
@@ -200,6 +212,22 @@ class FollowerService
         });
     }
 
+    public static function mutualAccounts($actorId, $profileId)
+    {
+        if($actorId == $profileId) {
+            return [];
+        }
+        $actorKey = self::FOLLOWING_KEY . $actorId;
+        $profileKey = self::FOLLOWERS_KEY . $profileId;
+        $key = self::FOLLOWERS_INTER_KEY . $actorId . ':' . $profileId;
+        $res = Redis::zinterstore($key, [$actorKey, $profileKey]);
+        if($res) {
+            return Redis::zrange($key, 0, -1);
+        } else {
+            return [];
+        }
+    }
+
     public static function delCache($id)
     {
         Redis::del(self::CACHE_KEY . $id);
@@ -208,4 +236,15 @@ class FollowerService
         Cache::forget(self::FOLLOWERS_SYNC_KEY . $id);
         Cache::forget(self::FOLLOWING_SYNC_KEY . $id);
     }
+
+    public static function localFollowerIds($pid, $limit = 0)
+    {
+        $key = self::FOLLOWERS_LOCAL_KEY . $pid;
+        $res = Cache::remember($key, 7200, function() use($pid) {
+            return DB::table('followers')->whereFollowingId($pid)->whereLocalProfile(true)->pluck('profile_id')->sort();
+        });
+        return $limit ?
+            $res->take($limit)->values()->toArray() :
+            $res->values()->toArray();
+    }
 }

app/Services/HashtagFollowService.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\HashtagFollow;
+
+class HashtagFollowService
+{
+	const FOLLOW_KEY = 'pf:services:hashtag-follows:v1:';
+	const CACHE_KEY = 'pf:services:hfs:byHid:';
+	const CACHE_WARMED = 'pf:services:hfs:wc:byHid';
+
+	public static function getPidByHid($hid)
+	{
+		if(!self::isWarm($hid)) {
+			return self::warmCache($hid);
+		}
+		return self::get($hid);
+	}
+
+	public static function unfollow($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function add($hid, $pid)
+	{
+		return Redis::zadd(self::CACHE_KEY . $hid, $pid, $pid);
+	}
+
+	public static function rem($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function get($hid)
+	{
+		return Redis::zrange(self::CACHE_KEY . $hid, 0, -1);
+	}
+
+	public static function count($hid)
+	{
+		return Redis::zcard(self::CACHE_KEY . $hid);
+	}
+
+	public static function warmCache($hid)
+	{
+		foreach(HashtagFollow::whereHashtagId($hid)->lazyById(20, 'id') as $h) {
+			if($h) {
+				self::add($h->hashtag_id, $h->profile_id);
+			}
+		}
+
+		self::setWarm($hid);
+
+		return self::get($hid);
+	}
+
+	public static function isWarm($hid)
+	{
+		return Redis::zcount(self::CACHE_KEY . $hid, 0, -1) ?? Redis::zscore(self::CACHE_WARMED, $hid) != null;
+	}
+
+	public static function setWarm($hid)
+	{
+		return Redis::zadd(self::CACHE_WARMED, $hid, $hid);
+	}
+}

app/Services/HashtagRelatedService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use DB;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+
+class HashtagRelatedService
+{
+    public static function get($id)
+    {
+        $tag = HashtagRelated::whereHashtagId($id)->first();
+        if(!$tag) {
+            return [];
+        }
+        return $tag->related_tags;
+    }
+
+    public static function fetchRelatedTags($tag)
+    {
+        $res = StatusHashtag::query()
+            ->select('h2.name', DB::raw('COUNT(*) as related_count'))
+            ->join('status_hashtags as hs2', function ($join) {
+                $join->on('status_hashtags.status_id', '=', 'hs2.status_id')
+                     ->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
+            })
+            ->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
+            ->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
+            ->where('h1.name', '=', $tag)
+            ->groupBy('h2.name')
+            ->orderBy('related_count', 'desc')
+            ->limit(30)
+            ->get();
+
+        return $res;
+    }
+}

app/Services/HashtagService.php

@@ -8,9 +8,10 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\HashtagFollow;
 
-class HashtagService {
-
-	const FOLLOW_KEY = 'pf:services:hashtag:following:';
+class HashtagService
+{
+    const FOLLOW_KEY = 'pf:services:hashtag:following:v1:';
+    const FOLLOW_PIDS_KEY = 'pf:services:hashtag-follows:v1:';
 
     public static function get($id)
     {
@@ -28,28 +29,29 @@ class HashtagService {
 
     public static function count($id)
     {
-		return Cache::remember('services:hashtag:public-count:by_id:' . $id, 86400, function() use($id) {
-			return StatusHashtag::whereHashtagId($id)->whereStatusVisibility('public')->count();
+        return Cache::remember('services:hashtag:total-count:by_id:' . $id, 300, function() use($id) {
+            $tag = Hashtag::find($id);
+            return $tag ? $tag->cached_count ?? 0 : 0;
         });
     }
 
     public static function isFollowing($pid, $hid)
     {
-		$res = Redis::zscore(self::FOLLOW_KEY . $pid, $hid);
+        $res = Redis::zscore(self::FOLLOW_KEY . $hid, $pid);
         if($res) {
             return true;
         }
 
-		$synced = Cache::get(self::FOLLOW_KEY . $pid . ':synced');
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct:' . $pid . ':synced');
         if(!$synced) {
             $tags = HashtagFollow::whereProfileId($pid)
                 ->get()
                 ->each(function($tag) use($pid) {
                     self::follow($pid, $tag->hashtag_id);
                 });
-			Cache::set(self::FOLLOW_KEY . $pid . ':synced', true, 1209600);
+            Cache::set(self::FOLLOW_KEY . 'acct:' . $pid . ':synced', true, 1209600);
 
-			return (bool) Redis::zscore(self::FOLLOW_KEY . $pid, $hid) > 1;
+            return (bool) Redis::zscore(self::FOLLOW_KEY . $hid, $pid) >= 1;
         }
 
         return false;
@@ -57,16 +59,29 @@ class HashtagService {
 
     public static function follow($pid, $hid)
     {
-		return Redis::zadd(self::FOLLOW_KEY . $pid, $hid, $hid);
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
+        return Redis::zadd(self::FOLLOW_KEY . $hid, $pid, $pid);
     }
 
     public static function unfollow($pid, $hid)
     {
-		return Redis::zrem(self::FOLLOW_KEY . $pid, $hid);
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
+        return Redis::zrem(self::FOLLOW_KEY . $hid, $pid);
     }
 
-	public static function following($pid, $start = 0, $limit = 10)
+    public static function following($hid, $start = 0, $limit = 10)
     {
-		return Redis::zrevrange(self::FOLLOW_KEY . $pid, $start, $limit);
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced');
+        if(!$synced) {
+            $tags = HashtagFollow::whereHashtagId($hid)
+                ->get()
+                ->each(function($tag) use($hid) {
+                    self::follow($tag->profile_id, $hid);
+                });
+            Cache::set(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced', true, 1209600);
+
+            return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
+        }
+        return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
     }
 }

app/Services/HomeTimelineService.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Follower;
+use App\Status;
+use App\Models\UserDomainBlock;
+
+class HomeTimelineService
+{
+    const CACHE_KEY = 'pf:services:timeline:home:';
+    const FOLLOWER_FEED_POST_LIMIT = 10;
+
+    public static function get($id, $start = 0, $stop = 10)
+    {
+        if($stop > 100) {
+            $stop = 100;
+        }
+
+        return Redis::zrevrange(self::CACHE_KEY . $id, $start, $stop);
+    }
+
+    public static function getRankedMaxId($id, $start = null, $limit = 10)
+    {
+        if(!$start) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, $start, '-inf', [
+            'withscores' => true,
+            'limit' => [1, $limit - 1]
+        ]));
+    }
+
+    public static function getRankedMinId($id, $end = null, $limit = 10)
+    {
+        if(!$end) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, '+inf', $end, [
+            'withscores' => true,
+            'limit' => [0, $limit]
+        ]));
+    }
+
+    public static function add($id, $val)
+    {
+        if(self::count($id) >= 400) {
+            Redis::zpopmin(self::CACHE_KEY . $id);
+        }
+
+        return Redis::zadd(self::CACHE_KEY .$id, $val, $val);
+    }
+
+    public static function rem($id, $val)
+    {
+        return Redis::zrem(self::CACHE_KEY . $id, $val);
+    }
+
+    public static function count($id)
+    {
+        return Redis::zcard(self::CACHE_KEY . $id);
+    }
+
+    public static function warmCache($id, $force = false, $limit = 100, $returnIds = false)
+    {
+        if(self::count($id) == 0 || $force == true) {
+            Redis::del(self::CACHE_KEY . $id);
+            $following = Cache::remember('profile:following:'.$id, 1209600, function() use($id) {
+                $following = Follower::whereProfileId($id)->pluck('following_id');
+                return $following->push($id)->toArray();
+            });
+
+            $minId = SnowflakeService::byDate(now()->subMonths(6));
+
+            $filters = UserFilterService::filters($id);
+
+            if($filters && count($filters)) {
+                $following = array_diff($following, $filters);
+            }
+
+            $domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray();
+
+            $ids = Status::where('id', '>', $minId)
+                ->whereIn('profile_id', $following)
+                ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+                ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+                ->whereIn('visibility',['public', 'unlisted', 'private'])
+                ->orderByDesc('id')
+                ->limit($limit)
+                ->pluck('id');
+
+            foreach($ids as $pid) {
+                $status = StatusService::get($pid, false);
+                if(!$status || !isset($status['account'], $status['url'])) {
+                    continue;
+                }
+                if($domainBlocks && count($domainBlocks)) {
+                    $domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
+                    if(in_array($domain, $domainBlocks)) {
+                        continue;
+                    }
+                }
+                self::add($id, $pid);
+            }
+
+            return $returnIds ? $ids : 1;
+        }
+        return 0;
+    }
+}

app/Services/InstanceService.php

@@ -120,6 +120,9 @@ class InstanceService
 				$pixels[] = $row;
 			}
 
+			// Free the allocated GdImage object from memory:
+			imagedestroy($image);
+
 			$components_x = 4;
 			$components_y = 4;
 			$blurhash = Blurhash::encode($pixels, $components_x, $components_y);

app/Services/LandingService.php

@@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis;
 use App\Status;
 use App\User;
 use App\Services\AccountService;
+use App\Util\Site\Nodeinfo;
 
 class LandingService
 {
 	public static function get($json = true)
 	{
-		$activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() {
-			return User::select('last_active_at')
-				->where('last_active_at', '>', now()->subMonths(1))
-				->orWhere('created_at', '>', now()->subMonths(1))
-				->count();
-		});
+		$activeMonth = Nodeinfo::activeUsersMonthly();
 
 		$totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() {
 			return User::count();

app/Services/MarkerService.php

@@ -13,7 +13,7 @@ class MarkerService
 		return Cache::get(self::CACHE_KEY . $timeline . ':' . $profileId);
 	}
 
-	public static function set($profileId, $timeline = 'home', $entityId)
+	public static function set($profileId, $timeline = 'home', $entityId = false)
 	{
 		$existing = self::get($profileId, $timeline);
 		$key = self::CACHE_KEY . $timeline . ':' . $profileId;

app/Services/Media/MediaHlsService.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services\Media;
+
+use Storage;
+
+class MediaHlsService
+{
+    public static function allFiles($media)
+    {
+        $path = $media->media_path;
+        if(!$path) { return; }
+        $parts = explode('/', $path);
+        $filename = array_pop($parts);
+        $dir = implode('/', $parts);
+        [$name, $ext] = explode('.', $filename);
+
+        $files = Storage::files($dir);
+
+        return collect($files)
+            ->filter(function($p) use($dir, $name) {
+                return str_starts_with($p, $dir . '/' . $name);
+            })
+            ->values()
+            ->toArray();
+    }
+}

app/Services/MediaService.php

@@ -18,7 +18,7 @@ class MediaService
 
 	public static function get($statusId)
 	{
-		return Cache::remember(self::CACHE_KEY.$statusId, 86400, function() use($statusId) {
+		return Cache::remember(self::CACHE_KEY.$statusId, 21600, function() use($statusId) {
 			$media = Media::whereStatusId($statusId)->orderBy('order')->get();
 			if(!$media) {
 				return [];
@@ -46,7 +46,8 @@ class MediaService
 				$media['orientation'],
 				$media['filter_name'],
 				$media['filter_class'],
-				$media['mime']
+				$media['mime'],
+				$media['hls_manifest']
 			);
 
 			$media['type'] = $mime ? strtolower($mime[0]) : 'unknown';

app/Services/MediaStorageService.php

@@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 use Illuminate\Support\Arr;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
 
 class MediaStorageService {
 
@@ -29,9 +30,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar, $local = false)
+	public static function avatar($avatar, $local = false, $skipRecentCheck = false)
 	{
-		return (new self())->fetchAvatar($avatar, $local);
+		return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
 	}
 
 	public static function head($url)
@@ -86,12 +87,11 @@ class MediaStorageService {
 		$thumbname = array_pop($pt);
 		$storagePath = implode('/', $p);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
-		$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
-		$url = $disk->url($file);
-		$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
-		$thumbUrl = $disk->url($thumbFile);
+		$url = ResilientMediaStorageService::store($storagePath, $path, $name);
+		if($thumb) {
+			$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
 			$media->thumbnail_url = $thumbUrl;
+		}
 		$media->cdn_url = $url;
 		$media->optimized_url = $url;
 		$media->replicated_at = now();
@@ -183,6 +183,7 @@ class MediaStorageService {
 
 	protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
 	{
+		$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
 		$url = $avatar->remote_url;
 		$driver = $local ? 'local' : config('filesystems.cloud');
 
@@ -206,7 +207,7 @@ class MediaStorageService {
 		$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
 
 		if(!$skipRecentCheck) {
-			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
+			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
 				return;
 			}
 		}
@@ -262,6 +263,7 @@ class MediaStorageService {
 
 		Cache::forget('avatar:' . $avatar->profile_id);
 		AccountService::del($avatar->profile_id);
+		AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
 
 		unlink($tmpName);
 	}

app/Services/NotificationService.php

@@ -12,10 +12,13 @@ use App\Transformer\Api\NotificationTransformer;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
 
 class NotificationService {
 
 	const CACHE_KEY = 'pf:services:notifications:ids:';
+	const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:';
+	const ITEM_CACHE_TTL = 86400;
 	const MASTODON_TYPES = [
 		'follow',
 		'follow_request',
@@ -44,11 +47,22 @@ class NotificationService {
 		return $res;
 	}
 
+	public static function getEpochId($months = 6)
+	{
+		$epoch = Cache::get(self::EPOCH_CACHE_KEY . $months);
+		if(!$epoch) {
+			NotificationEpochUpdatePipeline::dispatch();
+			return 1;
+		}
+		return $epoch;
+	}
+
 	public static function coldGet($id, $start = 0, $stop = 400)
 	{
 		$stop = $stop > 400 ? 400 : $stop;
-		$ids = Notification::whereProfileId($id)
-			->latest()
+		$ids = Notification::where('id', '>', self::getEpochId())
+			->where('profile_id', $id)
+			->orderByDesc('id')
 			->skip($start)
 			->take($stop)
 			->pluck('id');
@@ -227,7 +241,7 @@ class NotificationService {
 
 	public static function getNotification($id)
 	{
-		$notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) {
+		$notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) {
 			$n = Notification::with('item')->find($id);
 
 			if(!$n) {
@@ -259,7 +273,7 @@ class NotificationService {
 
 	public static function setNotification(Notification $notification)
 	{
-		return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
+		return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) {
 			$fractal = new Fractal\Manager();
 			$fractal->setSerializer(new ArraySerializer());
 			$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
@@ -270,8 +284,9 @@ class NotificationService {
 	public static function warmCache($id, $stop = 400, $force = false)
 	{
 		if(self::count($id) == 0 || $force == true) {
-			$ids = Notification::whereProfileId($id)
-				->latest()
+			$ids = Notification::where('profile_id', $id)
+				->where('id', '>', self::getEpochId())
+				->orderByDesc('id')
 				->limit($stop)
 				->pluck('id');
 			foreach($ids as $key) {

app/Services/PublicTimelineService.php

@@ -95,7 +95,7 @@ class PublicTimelineService {
 		if(self::count() == 0 || $force == true) {
 			$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
 			Redis::del(self::CACHE_KEY);
-			$minId = SnowflakeService::byDate(now()->subDays(14));
+			$minId = SnowflakeService::byDate(now()->subDays(90));
 			$ids = Status::where('id', '>', $minId)
 				->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id'])
 				->when($hideNsfw, function($q, $hideNsfw) {
@@ -105,9 +105,11 @@ class PublicTimelineService {
 				->whereScope('public')
 				->orderByDesc('id')
 				->limit($limit)
-				->pluck('id');
-			foreach($ids as $id) {
+				->pluck('id', 'profile_id');
+			foreach($ids as $k => $id) {
+                if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) {
 			         self::add($id);
+                }
 			}
 			return 1;
 		}

app/Services/RelationshipService.php

@@ -66,6 +66,14 @@ class RelationshipService
 		return self::get($aid, $tid);
 	}
 
+	public static function forget($aid, $tid)
+	{
+		Cache::forget('pf:services:follower:audience:' . $aid);
+		Cache::forget('pf:services:follower:audience:' . $tid);
+		self::delete($tid, $aid);
+		self::delete($aid, $tid);
+	}
+
 	public static function defaultRelation($tid)
 	{
 		return [

app/Services/ResilientMediaStorageService.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services;
+
+use Storage;
+use Illuminate\Http\File;
+use Exception;
+use GuzzleHttp\Exception\ClientException;
+use Aws\S3\Exception\S3Exception;
+use GuzzleHttp\Exception\ConnectException;
+use League\Flysystem\UnableToWriteFile;
+
+class ResilientMediaStorageService
+{
+    static $attempts = 0;
+
+    public static function store($storagePath, $path, $name)
+    {
+        return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
+            self::handleResilientStore($storagePath, $path, $name) :
+            self::handleStore($storagePath, $path, $name);
+    }
+
+    public static function handleStore($storagePath, $path, $name)
+    {
+        return retry(3, function() use($storagePath, $path, $name) {
+            $baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
+            $disk = Storage::disk($baseDisk);
+            $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            return $disk->url($file);
+        }, random_int(100, 500));
+    }
+
+    public static function handleResilientStore($storagePath, $path, $name)
+    {
+        $attempts = 0;
+        return retry(4, function() use($storagePath, $path, $name, $attempts) {
+            self::$attempts++;
+            usleep(100000);
+            $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
+            try {
+                $disk = Storage::disk($baseDisk);
+                $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
+            return $disk->url($file);
+        }, function (int $attempt, Exception $exception) {
+            return $attempt * 200;
+        });
+    }
+
+    public static function getAltDriver()
+    {
+        $drivers = [];
+        if(config('filesystems.disks.alt-primary.enabled')) {
+            $drivers[] = 'alt-primary';
+        }
+        if(config('filesystems.disks.alt-secondary.enabled')) {
+            $drivers[] = 'alt-secondary';
+        }
+        if(empty($drivers)) {
+            return false;
+        }
+        $key = array_rand($drivers, 1);
+        return $drivers[$key];
+    }
+}

app/Services/SearchApiV2Service.php

@@ -95,7 +95,15 @@ class SearchApiV2Service
         if(substr($webfingerQuery, 0, 1) !== '@') {
             $webfingerQuery = '@' . $webfingerQuery;
         }
-        $banned = InstanceService::getBannedDomains();
+        $banned = InstanceService::getBannedDomains() ?? [];
+        $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
+        if($domainBlocks && count($domainBlocks)) {
+            $banned = array_unique(
+                array_values(
+                    array_merge($banned, $domainBlocks)
+                )
+            );
+        }
         $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like';
         $results = Profile::select('username', 'id', 'followers_count', 'domain')
             ->where('username', $operator, $query)
@@ -172,8 +180,18 @@ class SearchApiV2Service
             'hashtags' => [],
             'statuses' => [],
         ];
+        $user = request()->user();
         $mastodonMode = self::$mastodonMode;
         $query = urldecode($this->query->input('q'));
+        $banned = InstanceService::getBannedDomains();
+        $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
+        if($domainBlocks && count($domainBlocks)) {
+            $banned = array_unique(
+                array_values(
+                    array_merge($banned, $domainBlocks)
+                )
+            );
+        }
         if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) {
             $default['accounts'] = $this->accounts(substr($query, 1));
             return $default;
@@ -197,7 +215,11 @@ class SearchApiV2Service
                 } catch (\Exception $e) {
                     return $default;
                 }
-                if($res && isset($res['id'])) {
+                if($res && isset($res['id'], $res['url'])) {
+                    $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
+                    if(in_array($domain, $banned)) {
+                        return $default;
+                    }
                     $default['accounts'][] = $res;
                     return $default;
                 } else {
@@ -212,6 +234,10 @@ class SearchApiV2Service
                     return $default;
                 }
                 if($res && isset($res['id'])) {
+                    $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
+                    if(in_array($domain, $banned)) {
+                        return $default;
+                    }
                     $default['accounts'][] = $res;
                     return $default;
                 } else {
@@ -221,6 +247,9 @@ class SearchApiV2Service
 
             if($sid = Status::whereUri($query)->first()) {
                 $s = StatusService::get($sid->id, false);
+                if(!$s) {
+                    return $default;
+                }
                 if(in_array($s['visibility'], ['public', 'unlisted'])) {
                     $default['statuses'][] = $s;
                     return $default;
@@ -229,7 +258,7 @@ class SearchApiV2Service
 
             try {
                 $res = ActivityPubFetchService::get($query);
-                $banned = InstanceService::getBannedDomains();
+
                 if($res) {
                     $json = json_decode($res, true);
 

app/Services/StatusHashtagService.php

@@ -84,10 +84,7 @@ class StatusHashtagService {
 
 	public static function statusTags($statusId)
 	{
-		$key = 'pf:services:sh:id:' . $statusId;
-
-		return Cache::remember($key, 604800, function() use($statusId) {
-			$status = Status::find($statusId);
+		$status = Status::with('hashtags')->find($statusId);
 		if(!$status) {
 			return [];
 		}
@@ -96,6 +93,5 @@ class StatusHashtagService {
 		$fractal->setSerializer(new ArraySerializer());
 		$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
 		return $fractal->createData($resource)->toArray();
-		});
 	}
 }