Merge branch 'pixelfed:dev' into dev

CHANGELOG.md

@@ -2,7 +2,41 @@
 
 ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
 
-## [v0.11.9 (2023-08-06)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)
+### Added
+- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
+- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
+- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
+
+### Federation
+- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
+- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
+
+### Updates
+- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59))
+- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904))
+- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce))
+- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d))
+- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec))
+- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d))
+- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973))
+- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7))
+- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605))
+- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27))
+- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
+- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
+- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
+- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
+- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
+- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
+- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
+- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
+- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
+- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed))
+- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
+
+## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)
 
 ### Added
 - Import from Instagram ([#4466](https://github.com/pixelfed/pixelfed/pull/4466)) ([cf3078c5](https://github.com/pixelfed/pixelfed/commit/cf3078c5))
@@ -57,8 +91,7 @@
 - Update RemoteStatusDelete pipeline ([71e92261](https://github.com/pixelfed/pixelfed/commit/71e92261))
 - Update RemoteStatusDelete pipeline ([fab8f25e](https://github.com/pixelfed/pixelfed/commit/fab8f25e))
 - Update RemoteStatusPipeline, fix reply check ([618b6727](https://github.com/pixelfed/pixelfed/commit/618b6727))
--  ([](https://github.com/pixelfed/pixelfed/commit/))
+- Update ApiV1Controller, add bookmarked to timeline entities ([ca746717](https://github.com/pixelfed/pixelfed/commit/ca746717))
--  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8)
 

app/Console/Commands/AvatarStorageDeepClean.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Cache;
+use Storage;
+use App\Avatar;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
+
+class AvatarStorageDeepClean extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'avatar:storage-deep-clean';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Cleanup avatar storage';
+
+    protected $shouldKeepRunning = true;
+    protected $counter = 0;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Pixelfed Avatar Deep Cleaner');
+        $this->line(' ');
+        $this->info('    Purge/delete old and outdated avatars from remote accounts');
+        $this->line(' ');
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
+            exit;
+        }
+
+        $start = 0;
+
+        if(!$this->confirm('Are you sure you want to proceed?')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        if(!$this->activeCheck()) {
+            $this->info('Found existing deep cleaning job');
+            if(!$this->confirm('Do you want to continue where you left off?')) {
+                $this->error('Aborting...');
+                exit;
+            } else {
+                $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
+
+                if($start && $start < 1 || $start > PHP_INT_MAX) {
+                    $this->error('Error fetching cached value');
+                    $this->error('Aborting...');
+                    exit;
+                }
+            }
+        }
+
+        $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
+        $bar = $this->output->createProgressBar($count);
+
+        foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
+            usleep(random_int(50, 1000));
+            $this->counter++;
+            $this->handleAvatar($avatar);
+            $bar->advance();
+        }
+        $bar->finish();
+    }
+
+    protected function updateCache($id)
+    {
+        Cache::put('cmd:asdp', $id);
+        if($this->counter % 5 === 0) {
+            Storage::put('avatar-deep-clean.json', $id);
+        }
+    }
+
+    protected function activeCheck()
+    {
+        if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    protected function handleAvatar($avatar)
+    {
+        $this->updateCache($avatar->id);
+        $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
+        $queue = $queues[random_int(0, 7)];
+        AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
+    }
+}

app/Console/Commands/UserAdmin.php

@@ -3,16 +3,17 @@
 namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
 use App\User;
 
-class UserAdmin extends Command
+class UserAdmin extends Command implements PromptsForMissingInput
 {
     /**
      * The name and signature of the console command.
      *
      * @var string
      */
-    protected $signature = 'user:admin {id}';
+    protected $signature = 'user:admin {username}';
 
     /**
      * The console command description.
@@ -22,13 +23,15 @@ class UserAdmin extends Command
     protected $description = 'Make a user an admin, or remove admin privileges.';
 
     /**
-     * Create a new command instance.
+     * Prompt for missing input arguments using the returned questions.
      *
-     * @return void
+     * @return array
      */
-    public function __construct()
+    protected function promptForMissingArgumentsUsing()
     {
-        parent::__construct();
+        return [
+            'username' => 'Which username should we toggle admin privileges for?',
+        ];
     }
 
     /**
@@ -38,16 +41,15 @@ class UserAdmin extends Command
      */
     public function handle()
     {
-        $id = $this->argument('id');
+        $id = $this->argument('username');
-        if(ctype_digit($id) == true) {
+
-            $user = User::find($id);
-        } else {
         $user = User::whereUsername($id)->first();
-        }
+
         if(!$user) {
             $this->error('Could not find any user with that username or id.');
             exit;
         }
+
         $this->info('Found username: ' . $user->username);
         $state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?';
         $confirmed = $this->confirm($state);

app/Console/Commands/UserToggle2FA.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use App\User;
+
+class UserToggle2FA extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'user:2fa {username}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Disable two factor authentication for given username';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'username' => 'Which username should we disable 2FA for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $user = User::whereUsername($this->argument('username'))->first();
+
+        if(!$user) {
+            $this->error('Could not find any user with that username');
+            exit;
+        }
+
+        if(!$user->{'2fa_enabled'}) {
+            $this->info('User did not have 2FA enabled!');
+            return;
+        }
+
+        $user->{'2fa_enabled'} = false;
+        $user->{'2fa_secret'} = null;
+        $user->{'2fa_backup_codes'} = null;
+        $user->save();
+
+        $this->info('Successfully disabled 2FA on this account!');
+    }
+}

app/Http/Controllers/Admin/AdminReportController.php

@@ -643,7 +643,7 @@ trait AdminReportController
 				$q->whereNull('admin_seen') :
 				$q->whereNotNull('admin_seen');
 			})
-			->groupBy(['object_id', 'object_type'])
+			->groupBy(['id', 'object_id', 'object_type', 'profile_id'])
 			->cursorPaginate(6)
 			->withQueryString()
 		);

app/Http/Controllers/AdminShadowFilterController.php

@@ -0,0 +1,122 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+use App\Models\AdminShadowFilter;
+use App\Profile;
+use App\Services\AccountService;
+use App\Services\AdminShadowFilterService;
+
+class AdminShadowFilterController extends Controller
+{
+    public function __construct()
+    {
+        $this->middleware(['auth','admin']);
+    }
+
+    public function home(Request $request)
+    {
+        $filter = $request->input('filter');
+        $searchQuery = $request->input('q');
+        $filters = AdminShadowFilter::when($filter, function($q, $filter) {
+            if($filter == 'all') {
+                return $q;
+            } else if($filter == 'inactive') {
+                return $q->whereActive(false);
+            } else {
+                return $q;
+            }
+        }, function($q, $filter) {
+            return $q->whereActive(true);
+        })
+        ->when($searchQuery, function($q, $searchQuery) {
+            $ids = Profile::where('username', 'like', '%' . $searchQuery . '%')
+                ->limit(100)
+                ->pluck('id')
+                ->toArray();
+            return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids);
+        })
+        ->latest()
+        ->paginate(10)
+        ->withQueryString();
+
+        return view('admin.asf.home', compact('filters'));
+    }
+
+    public function create(Request $request)
+    {
+        return view('admin.asf.create');
+    }
+
+    public function edit(Request $request, $id)
+    {
+        $filter = AdminShadowFilter::findOrFail($id);
+        $profile = AccountService::get($filter->item_id);
+        return view('admin.asf.edit', compact('filter', 'profile'));
+    }
+
+    public function store(Request $request)
+    {
+        $this->validate($request, [
+            'username' => 'required',
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $profile = Profile::whereUsername($request->input('username'))->first();
+
+        if(!$profile) {
+            return back()->withErrors(['Invalid account']);
+        }
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active') && $request->has('hide_from_public_feeds');
+
+        AdminShadowFilter::updateOrCreate([
+            'item_id' => $profile->id,
+            'item_type' => get_class($profile)
+        ], [
+            'is_local' => $profile->domain === null,
+            'note' => $request->input('note'),
+            'hide_from_public_feeds' => $request->has('hide_from_public_feeds'),
+            'admin_id' => $request->user()->profile_id,
+            'active' => $active
+        ]);
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+
+    public function storeEdit(Request $request, $id)
+    {
+        $this->validate($request, [
+            'active' => 'sometimes',
+            'note' => 'sometimes',
+            'hide_from_public_feeds' => 'sometimes'
+        ]);
+
+        $filter = AdminShadowFilter::findOrFail($id);
+
+        $profile = Profile::findOrFail($filter->item_id);
+
+        if($profile->user && $profile->user->is_admin) {
+            return back()->withErrors(['Cannot filter an admin account']);
+        }
+
+        $active = $request->has('active');
+        $filter->active = $active;
+        $filter->hide_from_public_feeds = $request->has('hide_from_public_feeds');
+        $filter->note = $request->input('note');
+        $filter->save();
+
+        AdminShadowFilterService::refresh();
+
+        return redirect('/i/admin/asf/home');
+    }
+}

app/Http/Controllers/Api/ApiV1Controller.php

@@ -2193,6 +2193,7 @@ class ApiV1Controller extends Controller
 				if($pid) {
 					$status['favourited'] = (bool) LikeService::liked($pid, $s['id']);
 					$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
+                    $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
 				}
 				return $status;
 			})
@@ -2203,6 +2204,7 @@ class ApiV1Controller extends Controller
                 if(!empty($status['reblog'])) {
                     $status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']);
                     $status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']);
+                    $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
                 }
 
                 return $status;
@@ -2244,6 +2246,7 @@ class ApiV1Controller extends Controller
 				if($pid) {
 					$status['favourited'] = (bool) LikeService::liked($pid, $s['id']);
 					$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
+                    $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
 				}
 				return $status;
 			})
@@ -2254,6 +2257,7 @@ class ApiV1Controller extends Controller
                 if(!empty($status['reblog'])) {
                     $status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']);
                     $status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']);
+                    $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
                 }
 
                 return $status;
@@ -2378,6 +2382,7 @@ class ApiV1Controller extends Controller
 			if($user) {
 				$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
 				$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $status['id']);
+                $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $status['id']);
 			}
 			return $status;
 		})
@@ -2502,7 +2507,7 @@ class ApiV1Controller extends Controller
 	{
 		abort_if(!$request->user(), 403);
 
-		$user = $request->user();
+		$pid = $request->user()->profile_id;
 
 		$res = $request->has(self::PF_API_ENTITY_KEY) ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
 		if(!$res || !isset($res['visibility'])) {
@@ -2512,17 +2517,23 @@ class ApiV1Controller extends Controller
 		$scope = $res['visibility'];
 		if(!in_array($scope, ['public', 'unlisted'])) {
 			if($scope === 'private') {
-				if(intval($res['account']['id']) !== intval($user->profile_id)) {
+				if(intval($res['account']['id']) !== intval($pid)) {
-					abort_unless(FollowerService::follows($user->profile_id, $res['account']['id']), 403);
+					abort_unless(FollowerService::follows($pid, $res['account']['id']), 403);
 				}
 			} else {
 				abort(400, 'Invalid request');
 			}
 		}
 
-		$res['favourited'] = LikeService::liked($user->profile_id, $res['id']);
+        if(!empty($res['reblog']) && isset($res['reblog']['id'])) {
-		$res['reblogged'] = ReblogService::get($user->profile_id, $res['id']);
+            $res['reblog']['favourited'] = (bool) LikeService::liked($pid, $res['reblog']['id']);
-		$res['bookmarked'] = BookmarkService::get($user->profile_id, $res['id']);
+            $res['reblog']['reblogged'] = (bool) ReblogService::get($pid, $res['reblog']['id']);
+            $res['reblog']['bookmarked'] = BookmarkService::get($pid, $res['reblog']['id']);
+        }
+
+		$res['favourited'] = LikeService::liked($pid, $res['id']);
+		$res['reblogged'] = ReblogService::get($pid, $res['id']);
+		$res['bookmarked'] = BookmarkService::get($pid, $res['id']);
 
 		return $this->json($res);
 	}
@@ -3615,8 +3626,8 @@ class ApiV1Controller extends Controller
 		abort_if(!$request->user(), 403);
 
 		$pid = $request->user()->profile_id;
-		$home = $request->input('home.last_read_id');
+		$home = $request->input('home[last_read_id]');
-		$notifications = $request->input('notifications.last_read_id');
+		$notifications = $request->input('notifications[last_read_id]');
 
 		if($home) {
 			return $this->json(MarkerService::set($pid, 'home', $home));

app/Http/Controllers/Api/ApiV2Controller.php

@@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{
 use App\Transformer\Api\{
 	RelationshipTransformer,
 };
+use App\Util\Site\Nodeinfo;
 
 class ApiV2Controller extends Controller
 {
@@ -77,12 +78,7 @@ class ApiV2Controller extends Controller
 			'description' => config_cache('app.short_description'),
 			'usage' => [
 				'users' => [
-					'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() {
+					'active_month' => (int) Nodeinfo::activeUsersMonthly()
-						return User::select('last_active_at', 'created_at')
-							->where('last_active_at', '>', now()->subMonths(1))
-							->orWhere('created_at', '>', now()->subMonths(1))
-							->count();
-					})
 				]
 			],
 			'thumbnail' => [

app/Http/Controllers/ComposeController.php

@@ -415,7 +415,7 @@ class ComposeController extends Controller
 		$results = Profile::select('id','domain','username')
 			->whereNotIn('id', $blocked)
 			->where('username','like','%'.$q.'%')
-			->groupBy('domain')
+			->groupBy('id', 'domain')
 			->limit(15)
 			->get()
 			->map(function($profile) {

app/Http/Controllers/RemoteAuthController.php

@@ -23,7 +23,13 @@ class RemoteAuthController extends Controller
 {
     public function start(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -37,7 +43,13 @@ class RemoteAuthController extends Controller
 
     public function getAuthDomains(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
 
         if(config('remote-auth.mastodon.domains.only_custom')) {
             $res = config('remote-auth.mastodon.domains.custom');
@@ -69,7 +81,14 @@ class RemoteAuthController extends Controller
 
     public function redirect(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $this->validate($request, ['domain' => 'required']);
 
         $domain = $request->input('domain');
@@ -158,6 +177,14 @@ class RemoteAuthController extends Controller
 
     public function preflight(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) {
             return redirect('/login');
         }
@@ -167,6 +194,14 @@ class RemoteAuthController extends Controller
 
     public function handleCallback(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
+
         $domain = $request->session()->get('oauth_domain');
 
         if($request->filled('code')) {
@@ -195,7 +230,13 @@ class RemoteAuthController extends Controller
 
     public function onboarding(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         if($request->user()) {
             return redirect('/');
         }
@@ -204,6 +245,13 @@ class RemoteAuthController extends Controller
 
     public function sessionCheck(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -248,6 +296,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonData(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -279,6 +334,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateUsername(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -334,6 +396,13 @@ class RemoteAuthController extends Controller
 
     public function sessionValidateEmail(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 403);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -359,6 +428,13 @@ class RemoteAuthController extends Controller
 
     public function sessionGetMastodonFollowers(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -386,6 +462,13 @@ class RemoteAuthController extends Controller
 
     public function handleSubmit(Request $request)
     {
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
         abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@@ -464,7 +547,13 @@ class RemoteAuthController extends Controller
 
     public function storeBio(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -483,7 +572,13 @@ class RemoteAuthController extends Controller
 
     public function accountToId(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@@ -525,7 +620,13 @@ class RemoteAuthController extends Controller
 
     public function storeAvatar(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
         $this->validate($request, [
             'avatar_url' => 'required|active_url',
@@ -547,7 +648,13 @@ class RemoteAuthController extends Controller
 
     public function finishUp(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_unless($request->user(), 404);
 
         $currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app');
@@ -564,7 +671,13 @@ class RemoteAuthController extends Controller
 
     public function handleLogin(Request $request)
     {
-        abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
+        abort_unless((
+            config_cache('pixelfed.open_registration') &&
+            config('remote-auth.mastodon.enabled')
+        ) || (
+            config('remote-auth.mastodon.ignore_closed_state') &&
+            config('remote-auth.mastodon.enabled')
+        ), 404);
         abort_if($request->user(), 404);
         abort_unless($request->session()->exists('oauth_domain'), 403);
         abort_unless($request->session()->exists('oauth_remote_session_token'), 403);

app/Http/Controllers/Settings/PrivacySettings.php

@@ -39,9 +39,11 @@ trait PrivacySettings
           'public_dm',
           'show_profile_follower_count',
           'show_profile_following_count',
+          'indexable',
           'show_atom',
         ];
 
+        $profile->indexable = $request->input('indexable') == 'on';
         $profile->is_suggestable = $request->input('is_suggestable') == 'on';
         $profile->save();
 
@@ -70,6 +72,8 @@ trait PrivacySettings
                 } else {
                     $settings->{$field} = false;
                 }
+            } elseif ($field == 'indexable') {
+
             } else {
                 if ($form == 'on') {
                     $settings->{$field} = true;

app/Http/Controllers/Stories/StoryApiV1Controller.php

@@ -20,6 +20,7 @@ use App\Jobs\StoryPipeline\StoryViewDeliver;
 use App\Services\AccountService;
 use App\Services\MediaPathService;
 use App\Services\StoryService;
+use App\Http\Resources\StoryView as StoryViewResource;
 
 class StoryApiV1Controller extends Controller
 {
@@ -355,4 +356,26 @@ class StoryApiV1Controller extends Controller
 		$path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
 		return $path;
 	}
+
+	public function viewers(Request $request)
+	{
+		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
+		$this->validate($request, [
+			'sid' => 'required|string|min:1|max:50'
+		]);
+
+		$pid = $request->user()->profile_id;
+		$sid = $request->input('sid');
+
+		$story = Story::whereProfileId($pid)
+			->whereActive(true)
+			->findOrFail($sid);
+
+		$viewers = StoryView::whereStoryId($story->id)
+            ->orderByDesc('id')
+			->cursorPaginate(10);
+
+		return StoryViewResource::collection($viewers);
+	}
 }

app/Http/Resources/StoryView.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Request;
+use Illuminate\Http\Resources\Json\JsonResource;
+use App\Services\AccountService;
+
+class StoryView extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @return array<string, mixed>
+     */
+    public function toArray(Request $request)
+    {
+        return AccountService::get($this->profile_id, true);
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+
+class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        AvatarService::cleanup($this->avatar, true);
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+use Illuminate\Support\Str;
+
+class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $avatar = $this->avatar;
+
+        $disk = AvatarService::disk();
+
+        $files = collect(AvatarService::storage($avatar));
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/CreateAvatar.php

@@ -2,19 +2,25 @@
 
 namespace App\Jobs\AvatarPipeline;
 
-use App\Avatar;
-use App\Profile;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Avatar;
+use App\Profile;
 
-class CreateAvatar implements ShouldQueue
+class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-    protected $profile;
+    public $profile;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
 
     /**
      * Delete the job if its models no longer exist.
@@ -23,6 +29,31 @@ class CreateAvatar implements ShouldQueue
      */
     public $deleteWhenMissingModels = true;
 
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:create:' . $this->profile->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
+    }
+    
     /**
      * Create a new job instance.
      *
@@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
      */
     public function __construct(Profile $profile)
     {
-        $this->profile = $profile;
+        $this->profile = $profile->withoutRelations();
     }
 
     /**
@@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
     public function handle()
     {
         $profile = $this->profile;
+        $isRemote = (bool) $profile->private_key == null;
         $path = 'public/avatars/default.jpg';
-        $avatar = new Avatar();
+        Avatar::updateOrCreate(
-        $avatar->profile_id = $profile->id;
+            [
-        $avatar->media_path = $path;
+                'profile_id' => $profile->id,
-        $avatar->change_count = 0;
+            ],
-        $avatar->last_processed_at = \Carbon\Carbon::now();
+            [
-        $avatar->save();
+                'media_path' => $path,
+                'change_count' => 0,
+                'is_remote' => $isRemote,
+                'last_processed_at' => now()
+            ]
+        );
     }
 }

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
+		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;
 	}

app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

@@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
 			$avatar->save();
 		}
 
-
 		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;

app/Jobs/FollowPipeline/FollowServiceWarmCache.php

@@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use Cache;
 use DB;
+use Storage;
+use App\Follower;
 use App\Profile;
 
 class FollowServiceWarmCache implements ShouldQueue
@@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue
 	public $timeout = 5000;
 	public $failOnTimeout = false;
 
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->profileId))->dontRelease()];
+    }
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue
 	{
 		$id = $this->profileId;
 
+        if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) {
+            return;
+        }
+
 		$account = AccountService::get($id, true);
 
 		if(!$account) {
@@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue
 			return;
 		}
 
-		DB::table('followers')
+        $hasFollowerPostProcessing = false;
-			->select('id', 'following_id', 'profile_id')
+        $hasFollowingPostProcessing = false;
-			->whereFollowingId($id)
-			->orderBy('id')
-			->chunk(200, function($followers) use($id) {
-			foreach($followers as $follow) {
-				FollowerService::add($follow->profile_id, $id);
-			}
-		});
 
-		DB::table('followers')
+        if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
-			->select('id', 'following_id', 'profile_id')
+            $following = [];
-			->whereProfileId($id)
+            $followers = [];
-			->orderBy('id')
+    		foreach(Follower::lazy() as $follow) {
-			->chunk(200, function($followers) use($id) {
+                if($follow->following_id != $id && $follow->profile_id != $id) {
-			foreach($followers as $follow) {
+                    continue;
-				FollowerService::add($id, $follow->following_id);
+                }
+                if($follow->profile_id == $id) {
+                    $following[] = $follow->following_id;
+                } else {
+                    $followers[] = $follow->profile_id;
+                }
+            }
+
+            if(count($followers) > 100) {
+                // store follower ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers));
+                $hasFollowerPostProcessing = true;
+            } else {
+                foreach($followers as $follower) {
+                    FollowerService::add($follower, $id);
+                }
+            }
+
+            if(count($following) > 100) {
+                // store following ids and process in another job
+                Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following));
+                $hasFollowingPostProcessing = true;
+            } else {
+                foreach($following as $following) {
+                    FollowerService::add($id, $following);
+                }
+            }
         }
-		});
 
 		Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800);
 		Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800);
@@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue
 
 		AccountService::del($id);
 
+        if($hasFollowingPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow');
+        }
+
+        if($hasFollowerPostProcessing) {
+            FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow');
+        }
+
 		return;
 	}
 }

app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Jobs\FollowPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\AccountService;
+use App\Services\FollowerService;
+use Cache;
+use DB;
+use Storage;
+use App\Follower;
+use App\Profile;
+
+class FollowServiceWarmCacheLargeIngestPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $profileId;
+    public $followType;
+    public $tries = 5;
+    public $timeout = 5000;
+    public $failOnTimeout = false;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($profileId, $followType = 'following')
+    {
+        $this->profileId = $profileId;
+        $this->followType = $followType;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $pid = $this->profileId;
+        $type = $this->followType;
+
+        if($type === 'followers') {
+            $key = 'follow-warm-cache/' . $pid . '/followers.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($id, $pid, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        if($type === 'following') {
+            $key = 'follow-warm-cache/' . $pid . '/following.json';
+            if(!Storage::exists($key)) {
+                return;
+            }
+            $file = Storage::get($key);
+            $json = json_decode($file, true);
+
+            foreach($json as $id) {
+                FollowerService::add($pid, $id, false);
+                usleep(random_int(500, 3000));
+            }
+            sleep(5);
+            Storage::delete($key);
+        }
+
+        sleep(random_int(2, 5));
+        $files = Storage::files('follow-warm-cache/' . $pid);
+        if(empty($files)) {
+            Storage::deleteDirectory('follow-warm-cache/' . $pid);
+        }
+    }
+}

app/Jobs/ProfilePipeline/DecrementPostCount.php

@@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue
             return 1;
         }
 
-        if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-            $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-            $profile->save();
-            AccountService::del($id);
-        } else {
         $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0;
         $profile->save();
         AccountService::del($id);
-        }
 
         return 1;
     }

app/Jobs/ProfilePipeline/IncrementPostCount.php

@@ -43,17 +43,10 @@ class IncrementPostCount implements ShouldQueue
 			return 1;
 		}
 
-		if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
-			$profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
-			$profile->last_status_at = now();
-			$profile->save();
-			AccountService::del($id);
-		} else {
 		$profile->status_count = $profile->status_count + 1;
 		$profile->last_status_at = now();
 		$profile->save();
 		AccountService::del($id);
-		}
 
 		return 1;
 	}

app/Jobs/StatusPipeline/RemoteStatusDelete.php

@@ -21,9 +21,11 @@ use App\{
 };
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use League\Fractal;
 use Illuminate\Support\Str;
 use League\Fractal\Serializer\ArraySerializer;
@@ -37,8 +39,9 @@ use App\Services\AccountService;
 use App\Services\CollectionService;
 use App\Services\StatusService;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
+use App\Jobs\ProfilePipeline\DecrementPostCount;
 
-class RemoteStatusDelete implements ShouldQueue
+class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -51,9 +54,35 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public $deleteWhenMissingModels = true;
 
-    public $timeout = 90;
+    public $tries = 3;
-    public $tries = 2;
+    public $maxExceptions = 3;
-    public $maxExceptions = 1;
+    public $timeout = 180;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'status:remote:delete:' . $this->status->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()];
+    }
 
     /**
      * Create a new job instance.
@@ -62,7 +91,7 @@ class RemoteStatusDelete implements ShouldQueue
      */
     public function __construct(Status $status)
     {
-        $this->status = $status;
+        $this->status = $status->withoutRelations();
     }
 
     /**
@@ -77,14 +106,10 @@ class RemoteStatusDelete implements ShouldQueue
         if($status->deleted_at) {
             return;
         }
-        $profile = $this->status->profile;
 
         StatusService::del($status->id, true);
 
-        if($profile->status_count && $profile->status_count > 0) {
+        DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox');
-            $profile->status_count = $profile->status_count - 1;
-            $profile->save();
-        }
 
         return $this->unlinkRemoveMedia($status);
     }

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -20,6 +20,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use App\Services\UserFilterService;
+use App\Services\AdminShadowFilterService;
 
 class StatusEntityLexer implements ShouldQueue
 {
@@ -176,7 +177,9 @@ class StatusEntityLexer implements ShouldQueue
 			$status->reblog_of_id === null &&
 			($hideNsfw ? $status->is_nsfw == false : true)
 		) {
+            if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
     			PublicTimelineService::add($status->id);
+            }
 		}
 
 		if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {

app/Jobs/StatusPipeline/StatusTagsPipeline.php

@@ -45,6 +45,11 @@ class StatusTagsPipeline implements ShouldQueue
 	{
 		$res = $this->activity;
 		$status = $this->status;
+
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
+
 		$tags = collect($res['tag']);
 
 		// Emoji
@@ -73,19 +78,18 @@ class StatusTagsPipeline implements ShouldQueue
 
             if(config('database.default') === 'pgsql') {
             	$hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name))
+            		->orWhere('slug', 'ilike', str_slug($name, '-', false))
             		->first();
 
 				if(!$hashtag) {
-            		$hashtag = new Hashtag;
+					$hashtag = Hashtag::updateOrCreate([
-            		$hashtag->name = $name;
+						'slug' => str_slug($name, '-', false),
-            		$hashtag->slug = str_slug($name);
+						'name' => $name
-            		$hashtag->save();
+					]);
 				}
             } else {
-				$hashtag = Hashtag::firstOrCreate([
+				$hashtag = Hashtag::updateOrCreate([
-					'slug' => str_slug($name)
+					'slug' => str_slug($name, '-', false),
-				], [
 					'name' => $name
 				]);
             }

app/Models/AdminShadowFilter.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use App\Services\AccountService;
+
+class AdminShadowFilter extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    protected $casts = [
+        'created_at' => 'datetime'
+    ];
+
+    public function account()
+    {
+        if($this->item_type === 'App\Profile') {
+            return AccountService::get($this->item_id, true);
+        }
+
+        return;
+    }
+}

app/Services/AdminShadowFilterService.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\AdminShadowFilter;
+use Cache;
+
+class AdminShadowFilterService
+{
+    const CACHE_KEY = 'pf:services:asfs:';
+
+    public static function queryFilter($name = 'hide_from_public_feeds')
+    {
+        return AdminShadowFilter::whereItemType('App\Profile')
+            ->whereActive(1)
+            ->where('hide_from_public_feeds', true)
+            ->pluck('item_id')
+            ->toArray();
+    }
+
+    public static function getHideFromPublicFeedsList($refresh = false)
+    {
+        $key = self::CACHE_KEY . 'list:hide_from_public_feeds';
+        if($refresh) {
+            Cache::forget($key);
+        }
+        return Cache::remember($key, 86400, function() {
+            return AdminShadowFilter::whereItemType('App\Profile')
+                ->whereActive(1)
+                ->where('hide_from_public_feeds', true)
+                ->pluck('item_id')
+                ->toArray();
+        });
+    }
+
+    public static function canAddToPublicFeedByProfileId($profileId)
+    {
+        return !in_array($profileId, self::getHideFromPublicFeedsList());
+    }
+
+    public static function refresh()
+    {
+        $keys = [
+            self::CACHE_KEY . 'list:hide_from_public_feeds'
+        ];
+
+        foreach($keys as $key) {
+            Cache::forget($key);
+        }
+    }
+}

app/Services/AvatarService.php

@@ -3,7 +3,13 @@
 namespace App\Services;
 
 use Cache;
+use Storage;
+use Illuminate\Support\Str;
+use App\Avatar;
 use App\Profile;
+use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
+use League\Flysystem\UnableToCheckDirectoryExistence;
+use League\Flysystem\UnableToRetrieveMetadata;
 
 class AvatarService
 {
@@ -20,4 +26,102 @@ class AvatarService
         }
         return $profile->avatarUrl();
     }
+
+    public static function disk()
+    {
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return false;
+        }
+
+        $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
+        $disk = Storage::disk($driver);
+
+        return $disk;
+    }
+
+    public static function storage(Avatar $avatar)
+    {
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        return $disk->allFiles($base . $avatar->profile_id);
+    }
+
+    public static function cleanup($avatar, $confirm = false)
+    {
+        if(!$avatar || !$confirm) {
+            return;
+        }
+
+        if($avatar->cdn_url == null) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return;
+        }
+
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        try {
+            $exists = $disk->directoryExists($base . $avatar->profile_id);
+        } catch (
+            UnableToRetrieveMetadata |
+            UnableToCheckDirectoryExistence |
+            Exception $e
+        ) {
+            return;
+        }
+
+        if(!$exists) {
+            return;
+        }
+
+        $files = collect($disk->allFiles($base . $avatar->profile_id));
+
+        if(!$files || !$files->count() || $files->count() === 1) {
+            return;
+        }
+
+        if($files->count() > 5) {
+            AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
+            return;
+        }
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
 }

app/Services/FollowerService.php

@@ -20,10 +20,14 @@ class FollowerService
 	const FOLLOWING_KEY = 'pf:services:follow:following:id:';
 	const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
 
-	public static function add($actor, $target)
+	public static function add($actor, $target, $refresh = true)
 	{
 		$ts = (int) microtime(true);
+        if($refresh) {
           RelationshipService::refresh($actor, $target);
+        } else {
+		  RelationshipService::forget($actor, $target);
+        }
 		Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target);
 		Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor);
 		Cache::forget('profile:following:' . $actor);

app/Services/InstanceService.php

@@ -120,6 +120,9 @@ class InstanceService
 				$pixels[] = $row;
 			}
 
+			// Free the allocated GdImage object from memory:
+			imagedestroy($image);
+
 			$components_x = 4;
 			$components_y = 4;
 			$blurhash = Blurhash::encode($pixels, $components_x, $components_y);

app/Services/LandingService.php

@@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis;
 use App\Status;
 use App\User;
 use App\Services\AccountService;
+use App\Util\Site\Nodeinfo;
 
 class LandingService
 {
 	public static function get($json = true)
 	{
-		$activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() {
+		$activeMonth = Nodeinfo::activeUsersMonthly();
-			return User::select('last_active_at')
-				->where('last_active_at', '>', now()->subMonths(1))
-				->orWhere('created_at', '>', now()->subMonths(1))
-				->count();
-		});
 
 		$totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() {
 			return User::count();

app/Services/MediaStorageService.php

@@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 use Illuminate\Support\Arr;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
 
 class MediaStorageService {
 
@@ -29,9 +30,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar, $local = false)
+	public static function avatar($avatar, $local = false, $skipRecentCheck = false)
 	{
-		return (new self())->fetchAvatar($avatar, $local);
+		return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
 	}
 
 	public static function head($url)
@@ -86,12 +87,11 @@ class MediaStorageService {
 		$thumbname = array_pop($pt);
 		$storagePath = implode('/', $p);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
+		$url = ResilientMediaStorageService::store($storagePath, $path, $name);
-		$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+		if($thumb) {
-		$url = $disk->url($file);
+			$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
-		$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
-		$thumbUrl = $disk->url($thumbFile);
 			$media->thumbnail_url = $thumbUrl;
+		}
 		$media->cdn_url = $url;
 		$media->optimized_url = $url;
 		$media->replicated_at = now();
@@ -183,6 +183,7 @@ class MediaStorageService {
 
 	protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
 	{
+		$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
 		$url = $avatar->remote_url;
 		$driver = $local ? 'local' : config('filesystems.cloud');
 
@@ -206,7 +207,7 @@ class MediaStorageService {
 		$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
 
 		if(!$skipRecentCheck) {
-			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
+			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
 				return;
 			}
 		}
@@ -262,6 +263,7 @@ class MediaStorageService {
 
 		Cache::forget('avatar:' . $avatar->profile_id);
 		AccountService::del($avatar->profile_id);
+		AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
 
 		unlink($tmpName);
 	}

app/Services/NotificationService.php

@@ -16,6 +16,8 @@ use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 class NotificationService {
 
 	const CACHE_KEY = 'pf:services:notifications:ids:';
+	const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:';
+	const ITEM_CACHE_TTL = 86400;
 	const MASTODON_TYPES = [
 		'follow',
 		'follow_request',
@@ -44,11 +46,22 @@ class NotificationService {
 		return $res;
 	}
 
+	public static function getEpochId($months = 6)
+	{
+		return Cache::remember(self::EPOCH_CACHE_KEY . $months, 1209600, function() use($months) {
+            if(Notification::count() === 0) {
+                return 0;
+            }
+			return Notification::where('created_at', '>', now()->subMonths($months))->first()->id;
+		});
+	}
+
 	public static function coldGet($id, $start = 0, $stop = 400)
 	{
 		$stop = $stop > 400 ? 400 : $stop;
-		$ids = Notification::whereProfileId($id)
+		$ids = Notification::where('id', '>', self::getEpochId())
-			->latest()
+			->where('profile_id', $id)
+			->orderByDesc('id')
 			->skip($start)
 			->take($stop)
 			->pluck('id');
@@ -227,7 +240,7 @@ class NotificationService {
 
 	public static function getNotification($id)
 	{
-		$notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) {
+		$notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) {
 			$n = Notification::with('item')->find($id);
 
 			if(!$n) {
@@ -259,7 +272,7 @@ class NotificationService {
 
 	public static function setNotification(Notification $notification)
 	{
-		return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
+		return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) {
 			$fractal = new Fractal\Manager();
 			$fractal->setSerializer(new ArraySerializer());
 			$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
@@ -270,8 +283,9 @@ class NotificationService {
 	public static function warmCache($id, $stop = 400, $force = false)
 	{
 		if(self::count($id) == 0 || $force == true) {
-			$ids = Notification::whereProfileId($id)
+			$ids = Notification::where('profile_id', $id)
-				->latest()
+				->where('id', '>', self::getEpochId())
+				->orderByDesc('id')
 				->limit($stop)
 				->pluck('id');
 			foreach($ids as $key) {

app/Services/PublicTimelineService.php

@@ -95,7 +95,7 @@ class PublicTimelineService {
 		if(self::count() == 0 || $force == true) {
 			$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
 			Redis::del(self::CACHE_KEY);
-			$minId = SnowflakeService::byDate(now()->subDays(14));
+			$minId = SnowflakeService::byDate(now()->subDays(90));
 			$ids = Status::where('id', '>', $minId)
 				->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id'])
 				->when($hideNsfw, function($q, $hideNsfw) {
@@ -105,9 +105,11 @@ class PublicTimelineService {
 				->whereScope('public')
 				->orderByDesc('id')
 				->limit($limit)
-				->pluck('id');
+				->pluck('id', 'profile_id');
-			foreach($ids as $id) {
+			foreach($ids as $k => $id) {
+                if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) {
 			         self::add($id);
+                }
 			}
 			return 1;
 		}

app/Services/RelationshipService.php

@@ -66,6 +66,14 @@ class RelationshipService
 		return self::get($aid, $tid);
 	}
 
+	public static function forget($aid, $tid)
+	{
+		Cache::forget('pf:services:follower:audience:' . $aid);
+		Cache::forget('pf:services:follower:audience:' . $tid);
+		self::delete($tid, $aid);
+		self::delete($aid, $tid);
+	}
+
 	public static function defaultRelation($tid)
 	{
 		return [

app/Services/ResilientMediaStorageService.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services;
+
+use Storage;
+use Illuminate\Http\File;
+use Exception;
+use GuzzleHttp\Exception\ClientException;
+use Aws\S3\Exception\S3Exception;
+use GuzzleHttp\Exception\ConnectException;
+use League\Flysystem\UnableToWriteFile;
+
+class ResilientMediaStorageService
+{
+    static $attempts = 0;
+
+    public static function store($storagePath, $path, $name)
+    {
+        return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
+            self::handleResilientStore($storagePath, $path, $name) :
+            self::handleStore($storagePath, $path, $name);
+    }
+
+    public static function handleStore($storagePath, $path, $name)
+    {
+        return retry(3, function() use($storagePath, $path, $name) {
+            $baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
+            $disk = Storage::disk($baseDisk);
+            $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            return $disk->url($file);
+        }, random_int(100, 500));
+    }
+
+    public static function handleResilientStore($storagePath, $path, $name)
+    {
+        $attempts = 0;
+        return retry(4, function() use($storagePath, $path, $name, $attempts) {
+            self::$attempts++;
+            usleep(100000);
+            $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
+            try {
+                $disk = Storage::disk($baseDisk);
+                $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
+            return $disk->url($file);
+        }, function (int $attempt, Exception $exception) {
+            return $attempt * 200;
+        });
+    }
+
+    public static function getAltDriver()
+    {
+        $drivers = [];
+        if(config('filesystems.disks.alt-primary.enabled')) {
+            $drivers[] = 'alt-primary';
+        }
+        if(config('filesystems.disks.alt-secondary.enabled')) {
+            $drivers[] = 'alt-secondary';
+        }
+        if(empty($drivers)) {
+            return false;
+        }
+        $key = array_rand($drivers, 1);
+        return $drivers[$key];
+    }
+}

app/Services/StatusService.php

@@ -22,9 +22,9 @@ class StatusService
         return self::CACHE_KEY . $p . $id;
     }
 
-    public static function get($id, $publicOnly = true)
+    public static function get($id, $publicOnly = true, $mastodonMode = false)
     {
-        return Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
+        $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
             if($publicOnly) {
                 $status = Status::whereScope('public')->find($id);
             } else {
@@ -36,13 +36,23 @@ class StatusService
             $fractal = new Fractal\Manager();
             $fractal->setSerializer(new ArraySerializer());
             $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
-            return $fractal->createData($resource)->toArray();
+            $res = $fractal->createData($resource)->toArray();
+            $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
+            if(isset($res['_pid'])) {
+                unset($res['account']);
+            }
+            return $res;
         });
+        if($res && isset($res['_pid'])) {
+            $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
+            unset($res['_pid']);
+        }
+        return $res;
     }
 
     public static function getMastodon($id, $publicOnly = true)
     {
-        $status = self::get($id, $publicOnly);
+        $status = self::get($id, $publicOnly, true);
         if(!$status) {
             return null;
         }
@@ -151,8 +161,6 @@ class StatusService
             }
             Cache::forget('status:transformer:media:attachments:' . $id);
             MediaService::del($id);
-            Cache::forget('status:thumb:nsfw0' . $id);
-            Cache::forget('status:thumb:nsfw1' . $id);
             Cache::forget('pf:services:sh:id:' . $id);
             PublicTimelineService::rem($id);
             NetworkTimelineService::rem($id);

app/Status.php

@@ -9,7 +9,9 @@ use App\Http\Controllers\StatusController;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use App\Models\Poll;
 use App\Services\AccountService;
+use App\Services\StatusService;
 use App\Models\StatusEdit;
+use Illuminate\Support\Str;
 
 class Status extends Model
 {
@@ -95,16 +97,30 @@ class Status extends Model
 
 	public function thumb($showNsfw = false)
 	{
-		$key = $showNsfw ? 'status:thumb:nsfw1'.$this->id : 'status:thumb:nsfw0'.$this->id;
+		$entity = StatusService::get($this->id, false);
-		return Cache::remember($key, now()->addMinutes(15), function() use ($showNsfw) {
+
-			$type = $this->type ?? $this->setType();
+		if(!$entity || !isset($entity['media_attachments']) || empty($entity['media_attachments'])) {
-			$is_nsfw = !$showNsfw ? $this->is_nsfw : false;
-			if ($this->media->count() == 0 || $is_nsfw || !in_array($type,['photo', 'photo:album', 'video'])) {
 			return url(Storage::url('public/no-preview.png'));
 		}
 
-			return url(Storage::url($this->firstMedia()->thumbnail_path));
+		if((!isset($entity['sensitive']) || $entity['sensitive']) && !$showNsfw) {
-		});
+			return url(Storage::url('public/no-preview.png'));
+		}
+
+        if(!isset($entity['visibility']) || !in_array($entity['visibility'], ['public', 'unlisted'])) {
+            return url(Storage::url('public/no-preview.png'));
+        }
+
+		return collect($entity['media_attachments'])
+            ->filter(fn($media) => $media['type'] == 'image' && in_array($media['mime'], ['image/jpeg', 'image/png']))
+            ->map(function($media) {
+                if(!Str::endsWith($media['preview_url'], ['no-preview.png', 'no-preview.jpg'])) {
+                    return $media['preview_url'];
+                }
+
+                return $media['url'];
+            })
+            ->first() ?? url(Storage::url('public/no-preview.png'));
 	}
 
 	public function url($forceLocal = false)

app/Transformer/ActivityPub/ProfileTransformer.php

@@ -15,6 +15,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract
             'https://w3id.org/security/v1',
             'https://www.w3.org/ns/activitystreams',
             [
+              'toot' => 'http://joinmastodon.org/ns#',
               'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers',
               'alsoKnownAs' => [
                     '@id' => 'as:alsoKnownAs',
@@ -23,7 +24,8 @@ class ProfileTransformer extends Fractal\TransformerAbstract
               'movedTo' => [
                     '@id' => 'as:movedTo',
                     '@type' => '@id'
-              ]
+              ],
+              'indexable' => 'toot:indexable',
             ],
           ],
           'id'                        => $profile->permalink(),
@@ -37,6 +39,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract
           'summary'                   => $profile->bio,
           'url'                       => $profile->url(),
           'manuallyApprovesFollowers' => (bool) $profile->is_private,
+          'indexable'                 => (bool) $profile->indexable,
           'publicKey' => [
             'id'           => $profile->permalink().'#main-key',
             'owner'        => $profile->permalink(),

app/Transformer/ActivityPub/Verb/CreateNote.php

@@ -81,7 +81,8 @@ class CreateNote extends Fractal\TransformerAbstract
 						'@type' 		=> '@id'
 					],
 					'toot' 				=> 'http://joinmastodon.org/ns#',
-					'Emoji'				=> 'toot:Emoji'
+					'Emoji'				=> 'toot:Emoji',
+					'blurhash'			=> 'toot:blurhash',
 				]
 			],
 			'id' 					=> $status->permalink(),
@@ -103,12 +104,22 @@ class CreateNote extends Fractal\TransformerAbstract
 				'cc' 				=> $status->scopeToAudience('cc'),
 				'sensitive'       	=> (bool) $status->is_nsfw,
 				'attachment'      	=> $status->media()->orderBy('order')->get()->map(function ($media) {
-					return [
+					$res = [
 						'type'      => $media->activityVerb(),
 						'mediaType' => $media->mime,
 						'url'       => $media->url(),
 						'name'      => $media->caption,
 					];
+					if($media->blurhash) {
+						$res['blurhash'] = $media->blurhash;
+					}
+					if($media->width) {
+						$res['width'] = $media->width;
+					}
+					if($media->height) {
+						$res['height'] = $media->height;
+					}
+					return $res;
 				})->toArray(),
 				'tag' 				=> $tags,
 				'commentsEnabled'  => (bool) !$status->comments_disabled,

app/Transformer/ActivityPub/Verb/Note.php

@@ -82,7 +82,8 @@ class Note extends Fractal\TransformerAbstract
 						'@type' 		=> '@id'
 					],
 					'toot' 				=> 'http://joinmastodon.org/ns#',
-					'Emoji'				=> 'toot:Emoji'
+					'Emoji'				=> 'toot:Emoji',
+					'blurhash'			=> 'toot:blurhash',
 				]
 			],
 			'id' 				=> $status->url(),
@@ -97,12 +98,22 @@ class Note extends Fractal\TransformerAbstract
 			'cc' 				=> $status->scopeToAudience('cc'),
 			'sensitive'       	=> (bool) $status->is_nsfw,
 			'attachment'      	=> $status->media()->orderBy('order')->get()->map(function ($media) {
-				return [
+				$res = [
 					'type'      => $media->activityVerb(),
 					'mediaType' => $media->mime,
 					'url'       => $media->url(),
 					'name'      => $media->caption,
 				];
+				if($media->blurhash) {
+					$res['blurhash'] = $media->blurhash;
+				}
+				if($media->width) {
+					$res['width'] = $media->width;
+				}
+				if($media->height) {
+					$res['height'] = $media->height;
+				}
+				return $res;
 			})->toArray(),
 			'tag' 				=> $tags,
 			'commentsEnabled'  => (bool) !$status->comments_disabled,

app/Transformer/Api/StatusStatelessTransformer.php

@@ -16,6 +16,7 @@ use App\Services\StatusLabelService;
 use App\Services\StatusMentionService;
 use App\Services\PollService;
 use App\Models\CustomEmoji;
+use App\Util\Lexer\Autolink;
 
 class StatusStatelessTransformer extends Fractal\TransformerAbstract
 {
@@ -23,6 +24,9 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract
 	{
 		$taggedPeople = MediaTagService::get($status->id);
 		$poll = $status->type === 'poll' ? PollService::get($status->id) : null;
+        $rendered = config('exp.autolink') ?
+            ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) :
+            ( $status->rendered ?? $status->caption );
 
 		return [
 			'_v'                        => 1,
@@ -34,7 +38,7 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract
 			'in_reply_to_id'            => $status->in_reply_to_id ? (string) $status->in_reply_to_id : null,
 			'in_reply_to_account_id'    => $status->in_reply_to_profile_id ? (string) $status->in_reply_to_profile_id : null,
 			'reblog'                    => $status->reblog_of_id ? StatusService::get($status->reblog_of_id, false) : null,
-			'content'                   => $status->rendered ?? $status->caption,
+			'content'                   => $rendered,
 			'content_text'              => $status->caption,
 			'created_at'                => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)),
 			'emojis'                    => CustomEmoji::scan($status->caption),

app/Transformer/Api/StatusTransformer.php

@@ -19,6 +19,7 @@ use Illuminate\Support\Str;
 use App\Services\PollService;
 use App\Models\CustomEmoji;
 use App\Services\BookmarkService;
+use App\Util\Lexer\Autolink;
 
 class StatusTransformer extends Fractal\TransformerAbstract
 {
@@ -27,6 +28,9 @@ class StatusTransformer extends Fractal\TransformerAbstract
 		$pid = request()->user()->profile_id;
 		$taggedPeople = MediaTagService::get($status->id);
 		$poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null;
+        $rendered = config('exp.autolink') ?
+            ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) :
+            ( $status->rendered ?? $status->caption );
 
 		return [
 			'_v'                        => 1,
@@ -37,7 +41,7 @@ class StatusTransformer extends Fractal\TransformerAbstract
 			'in_reply_to_id'            => (string) $status->in_reply_to_id,
 			'in_reply_to_account_id'    => (string) $status->in_reply_to_profile_id,
 			'reblog'                    => $status->reblog_of_id ? StatusService::get($status->reblog_of_id) : null,
-			'content'                   => $status->rendered ?? $status->caption,
+			'content'                   => $rendered,
 			'content_text'              => $status->caption,
 			'created_at'                => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)),
 			'emojis'                    => CustomEmoji::scan($status->caption),

app/Util/ActivityPub/Helpers.php

@@ -108,7 +108,10 @@ class Helpers {
                 'string',
                 Rule::in($mimeTypes)
             ],
-            '*.name' => 'sometimes|nullable|string'
+            '*.name' => 'sometimes|nullable|string',
+            '*.blurhash' => 'sometimes|nullable|string|min:6|max:164',
+            '*.width' => 'sometimes|nullable|integer|min:1|max:5000',
+            '*.height' => 'sometimes|nullable|integer|min:1|max:5000',
         ])->passes();
 
         return $valid;
@@ -276,7 +279,7 @@ class Helpers {
         }
 
         if(is_array($val)) {
-            return !empty($val) ? $val[0] : null;
+            return !empty($val) ? head($val) : null;
         }
 
         return null;
@@ -684,6 +687,8 @@ class Helpers {
             $blurhash = isset($media['blurhash']) ? $media['blurhash'] : null;
             $license = isset($media['license']) ? License::nameToId($media['license']) : null;
             $caption = isset($media['name']) ? Purify::clean($media['name']) : null;
+            $width = isset($media['width']) ? $media['width'] : false;
+            $height = isset($media['height']) ? $media['height'] : false;
 
             $media = new Media();
             $media->blurhash = $blurhash;
@@ -695,6 +700,12 @@ class Helpers {
             $media->remote_url = $url;
             $media->caption = $caption;
             $media->order = $key + 1;
+            if($width) {
+                $media->width = $width;
+            }
+            if($height) {
+                $media->height = $height;
+            }
             if($license) {
                 $media->license = $license;
             }
@@ -785,11 +796,12 @@ class Helpers {
                 'inbox_url' => $res['inbox'],
                 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null,
                 'public_key' => $res['publicKey']['publicKeyPem'],
+                'indexable' => isset($res['indexable']) && is_bool($res['indexable']) ? $res['indexable'] : false,
             ]
         );
 
         if( $profile->last_fetched_at == null ||
-            $profile->last_fetched_at->lt(now()->subHours(24))
+            $profile->last_fetched_at->lt(now()->subMonths(3))
         ) {
             RemoteAvatarFetch::dispatch($profile);
         }

app/Util/Lexer/Regex.php

@@ -162,7 +162,7 @@ abstract class Regex
         //      look-ahead capture here and don't append $after when we return.
         $tmp['valid_mention_preceding_chars'] = '([^a-zA-Z0-9_!#\$%&*@＠\/]|^|(?:^|[^a-z0-9_+~.-])RT:?)';
 
-        $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([a-z0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i';
+        $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([\p{L}0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/iu';
 
         $re['valid_reply'] = '/^(?:['.$tmp['spaces'].'])*['.$tmp['at_signs'].']([a-z0-9_\-.]{1,20})(?=(.*|$))/iu';
         $re['end_mention_match'] = '/\A(?:['.$tmp['at_signs'].']|['.$tmp['latin_accents'].']|:\/\/)/iu';

app/Util/Media/Blurhash.php

@@ -44,6 +44,9 @@ class Blurhash {
 			$pixels[] = $row;
 		}
 
+		// Free the allocated GdImage object from memory:
+		imagedestroy($image);
+
 		$components_x = 4;
 		$components_y = 4;
 		$blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y);

app/Util/Site/Nodeinfo.php

@@ -2,28 +2,20 @@
 
 namespace App\Util\Site;
 
-use Cache;
+use Illuminate\Support\Facades\Cache;
-use App\{Like, Profile, Status, User};
+use App\Like;
+use App\Profile;
+use App\Status;
+use App\User;
 use Illuminate\Support\Str;
 
-class Nodeinfo {
+class Nodeinfo
-
+{
     public static function get()
     {
-		$res = Cache::remember('api:nodeinfo', 300, function () {
+        $res = Cache::remember('api:nodeinfo', 900, function () {
-			$activeHalfYear = Cache::remember('api:nodeinfo:ahy', 172800, function() {
+            $activeHalfYear = self::activeUsersHalfYear();
-				return User::select('last_active_at')
+            $activeMonth = self::activeUsersMonthly();
-					->where('last_active_at', '>', now()->subMonths(6))
-					->orWhere('created_at', '>', now()->subMonths(6))
-					->count();
-			});
-
-			$activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() {
-				return User::select('last_active_at')
-					->where('last_active_at', '>', now()->subMonths(1))
-					->orWhere('created_at', '>', now()->subMonths(1))
-					->count();
-			});
 
             $users = Cache::remember('api:nodeinfo:users', 43200, function() {
                 return User::count();
@@ -83,4 +75,25 @@ class Nodeinfo {
         ];
     }
 
+    public static function activeUsersMonthly()
+    {
+        return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
+            return User::withTrashed()
+                    ->select('last_active_at, updated_at')
+                    ->where('updated_at', '>', now()->subWeeks(5))
+                    ->orWhere('last_active_at', '>', now()->subWeeks(5))
+                    ->count();
+        });
+    }
+
+    public static function activeUsersHalfYear()
+    {
+        return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
+            return User::withTrashed()
+                ->select('last_active_at, updated_at')
+                ->where('last_active_at', '>', now()->subMonths(6))
+                ->orWhere('updated_at', '>', now()->subMonths(6))
+                ->count();
+        });
+    }
 }

config/exp.php

@@ -41,4 +41,6 @@ return [
 
 	// Post Update/Edits
 	'pue' => env('EXP_PUE', true),
+
+	'autolink' => env('EXP_AUTOLINK_V2', false),
 ];

config/filesystems.php

@@ -79,6 +79,34 @@ return [
             'throw' => true,
         ],
 
+        'alt-primary' => [
+            'enabled'  => env('ALT_PRI_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_PRI_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_PRI_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_PRI_AWS_URL'),
+            'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
+        'alt-secondary' => [
+            'enabled'  => env('ALT_SEC_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_SEC_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_SEC_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_SEC_AWS_URL'),
+            'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
         'spaces' => [
             'driver' => 's3',
             'key' => env('DO_SPACES_KEY'),

config/media.php

@@ -18,7 +18,9 @@ return [
             | Disabled by default.
             |
             */
-			'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false)
+            'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
+
+            'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
         ],
     ]
 ];

config/pixelfed.php

@@ -23,7 +23,7 @@ return [
 	| This value is the version of your Pixelfed instance.
 	|
 	*/
-	'version' => '0.11.8',
+	'version' => '0.11.9',
 
 	/*
 	|--------------------------------------------------------------------------

config/remote-auth.php

@@ -3,6 +3,7 @@
 return [
     'mastodon' => [
         'enabled' => env('PF_LOGIN_WITH_MASTODON_ENABLED', false),
+        'ignore_closed_state' => env('PF_LOGIN_WITH_MASTODON_ENABLED_SKIP_CLOSED', false),
 
         'contraints' => [
             /*

database/migrations/2021_08_04_095125_create_groups_table.php

@@ -0,0 +1,42 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGroupsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('groups', function (Blueprint $table) {
+            $table->bigInteger('id')->unsigned()->primary();
+            $table->bigInteger('profile_id')->unsigned()->nullable()->index();
+            $table->string('status')->nullable()->index();
+            $table->string('name')->nullable();
+            $table->text('description')->nullable();
+            $table->text('rules')->nullable();
+            $table->boolean('local')->default(true)->index();
+            $table->string('remote_url')->nullable();
+            $table->string('inbox_url')->nullable();
+            $table->boolean('is_private')->default(false);
+            $table->boolean('local_only')->default(false);
+            $table->json('metadata')->nullable();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('groups');
+    }
+}

database/migrations/2021_08_04_095143_create_group_members_table.php

@@ -0,0 +1,40 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGroupMembersTable extends Migration
+{
+	/**
+	 * Run the migrations.
+	 *
+	 * @return void
+	 */
+	public function up()
+	{
+		Schema::create('group_members', function (Blueprint $table) {
+			$table->id();
+			$table->bigInteger('group_id')->unsigned()->index();
+			$table->bigInteger('profile_id')->unsigned()->index();
+			$table->string('role')->default('member')->index();
+			$table->boolean('local_group')->default(false)->index();
+			$table->boolean('local_profile')->default(false)->index();
+			$table->boolean('join_request')->default(false)->index();
+			$table->timestamp('approved_at')->nullable();
+			$table->timestamp('rejected_at')->nullable();
+			$table->unique(['group_id', 'profile_id']);
+			$table->timestamps();
+		});
+	}
+
+	/**
+	 * Reverse the migrations.
+	 *
+	 * @return void
+	 */
+	public function down()
+	{
+		Schema::dropIfExists('group_members');
+	}
+}

database/migrations/2021_08_04_095238_create_group_posts_table.php

@@ -0,0 +1,42 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGroupPostsTable extends Migration
+{
+	/**
+	 * Run the migrations.
+	 *
+	 * @return void
+	 */
+	public function up()
+	{
+		Schema::create('group_posts', function (Blueprint $table) {
+			$table->bigInteger('id')->unsigned()->primary();
+			$table->bigInteger('group_id')->unsigned()->index();
+			$table->bigInteger('profile_id')->unsigned()->nullable()->index();
+			$table->string('type')->nullable()->index();
+			$table->bigInteger('status_id')->unsigned()->unique();
+			$table->string('remote_url')->unique()->nullable()->index();
+			$table->bigInteger('reply_child_id')->unsigned()->nullable();
+			$table->bigInteger('in_reply_to_id')->unsigned()->nullable();
+			$table->bigInteger('reblog_of_id')->unsigned()->nullable();
+			$table->unsignedInteger('reply_count')->nullable();
+			$table->string('status')->nullable()->index();
+			$table->json('metadata')->nullable();
+			$table->timestamps();
+		});
+	}
+
+	/**
+	 * Reverse the migrations.
+	 *
+	 * @return void
+	 */
+	public function down()
+	{
+		Schema::dropIfExists('group_posts');
+	}
+}

database/migrations/2021_08_16_072457_create_group_invitations_table.php

@@ -0,0 +1,38 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGroupInvitationsTable extends Migration
+{
+	/**
+	 * Run the migrations.
+	 *
+	 * @return void
+	 */
+	public function up()
+	{
+		Schema::create('group_invitations', function (Blueprint $table) {
+			$table->bigIncrements('id');
+			$table->bigInteger('group_id')->unsigned()->index();
+			$table->bigInteger('from_profile_id')->unsigned()->index();
+			$table->bigInteger('to_profile_id')->unsigned()->index();
+			$table->string('role')->nullable();
+			$table->boolean('to_local')->default(true)->index();
+			$table->boolean('from_local')->default(true)->index();
+			$table->unique(['group_id', 'to_profile_id']);
+			$table->timestamps();
+		});
+	}
+
+	/**
+	 * Reverse the migrations.
+	 *
+	 * @return void
+	 */
+	public function down()
+	{
+		Schema::dropIfExists('group_invitations');
+	}
+}

database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('profiles', function (Blueprint $table) {
+            $table->boolean('indexable')->default(false)->index();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('profiles', function (Blueprint $table) {
+            $table->dropColumn('indexable');
+        });
+    }
+};

database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php

@@ -0,0 +1,47 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('admin_shadow_filters', function (Blueprint $table) {
+            $table->id();
+            $table->unsignedBigInteger('admin_id')->nullable();
+            $table->morphs('item');
+            $table->boolean('is_local')->default(true)->index();
+            $table->text('note')->nullable();
+            $table->boolean('active')->default(false)->index();
+            $table->json('history')->nullable();
+            $table->json('ruleset')->nullable();
+            $table->boolean('prevent_ap_fanout')->default(false)->index();
+            $table->boolean('prevent_new_dms')->default(false)->index();
+            $table->boolean('ignore_reports')->default(false)->index();
+            $table->boolean('ignore_mentions')->default(false)->index();
+            $table->boolean('ignore_links')->default(false)->index();
+            $table->boolean('ignore_hashtags')->default(false)->index();
+            $table->boolean('hide_from_public_feeds')->default(false)->index();
+            $table->boolean('hide_from_tag_feeds')->default(false)->index();
+            $table->boolean('hide_embeds')->default(false)->index();
+            $table->boolean('hide_from_story_carousel')->default(false)->index();
+            $table->boolean('hide_from_search_autocomplete')->default(false)->index();
+            $table->boolean('hide_from_search')->default(false)->index();
+            $table->boolean('requires_login')->default(false)->index();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('admin_shadow_filters');
+    }
+};

resources/views/admin/asf/create.blade.php

@@ -0,0 +1,64 @@
+@extends('admin.partial.template-full')
+
+@section('section')
+</div><div class="header bg-primary pb-3 mt-n4">
+    <div class="container-fluid">
+        <div class="header-body">
+            <div class="row align-items-center py-4">
+                <div class="col-lg-6 col-7">
+                    <p class="display-1 text-white d-inline-block mb-0">New Shadow Filters</p>
+                    <p class="text-white mb-0">Creating a new admin shadow filter</p>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+    <div class="m-n2 m-lg-4">
+        <div class="container-fluid mt-4">
+            <div class="row justify-content-center">
+                <div class="col-12 col-md-6">
+                    @if ($errors->any())
+                    <div class="alert alert-danger">
+                        <ul class="mb-0">
+                            @foreach ($errors->all() as $error)
+                            <li class="mb-0 font-weight-bold">{{ $error }}</li>
+                            @endforeach
+                        </ul>
+                    </div>
+                    @endif
+                    <div class="card card-body">
+                        <form method="post">
+                            @csrf
+                            <div class="form-group">
+                                <label class="font-weight-bold">Username</label>
+                                <input class="form-control" name="username" placeholder="Enter username here" />
+                            </div>
+
+                            <p class="mb-0 font-weight-bold small">Filters</p>
+                            <div class="list-group mb-3">
+                                <div class="list-group-item">
+                                    <div class="custom-control custom-checkbox">
+                                        <input type="checkbox" class="custom-control-input" id="hide_from_public_feeds" name="hide_from_public_feeds">
+                                        <label class="custom-control-label" for="hide_from_public_feeds">Hide public posts from public feed</label>
+                                    </div>
+                                </div>
+                                {{-- <div class="list-group-item"></div> --}}
+                            </div>
+
+                            <div class="form-group">
+                                <label class="font-weight-bold">Note</label>
+                                <textarea class="form-control" name="note" placeholder="Add an optional note, only visible to admins"></textarea>
+                            </div>
+                            <div class="custom-control custom-checkbox">
+                                <input type="checkbox" class="custom-control-input" id="active" name="active" checked>
+                                <label class="custom-control-label font-weight-bold" for="active">Mark as Active</label>
+                            </div>
+                            <hr>
+                            <button type="submit" class="btn btn-success">Save</button>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+@endsection

resources/views/admin/asf/edit.blade.php

@@ -0,0 +1,64 @@
+@extends('admin.partial.template-full')
+
+@section('section')
+</div><div class="header bg-primary pb-3 mt-n4">
+    <div class="container-fluid">
+        <div class="header-body">
+            <div class="row align-items-center py-4">
+                <div class="col-lg-6 col-7">
+                    <p class="display-1 text-white d-inline-block mb-0">Edit Shadow Filters</p>
+                    <p class="text-white mb-0">Editing shadow filters</p>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+    <div class="m-n2 m-lg-4">
+        <div class="container-fluid mt-4">
+            <div class="row justify-content-center">
+                <div class="col-12 col-md-6">
+                    @if ($errors->any())
+                    <div class="alert alert-danger">
+                        <ul class="mb-0">
+                            @foreach ($errors->all() as $error)
+                            <li class="mb-0 font-weight-bold">{{ $error }}</li>
+                            @endforeach
+                        </ul>
+                    </div>
+                    @endif
+                    <div class="card card-body">
+                        <form method="post">
+                            @csrf
+                            <div class="form-group">
+                                <label class="font-weight-bold">Username</label>
+                                <input class="form-control" name="username" placeholder="Enter username here" value="{{ $profile['username'] }}" disabled="disabled" />
+                            </div>
+
+                            <p class="mb-0 font-weight-bold small">Filters</p>
+                            <div class="list-group mb-3">
+                                <div class="list-group-item">
+                                    <div class="custom-control custom-checkbox">
+                                        <input type="checkbox" class="custom-control-input" id="hide_from_public_feeds" name="hide_from_public_feeds" {!! $filter->hide_from_public_feeds ? 'checked=""' : '' !!}>
+                                        <label class="custom-control-label" for="hide_from_public_feeds">Hide public posts from public feed</label>
+                                    </div>
+                                </div>
+                                {{-- <div class="list-group-item"></div> --}}
+                            </div>
+
+                            <div class="form-group">
+                                <label class="font-weight-bold">Note</label>
+                                <textarea class="form-control" name="note" placeholder="Add an optional note, only visible to admins">{{ $filter->note }}</textarea>
+                            </div>
+                            <div class="custom-control custom-checkbox">
+                                <input type="checkbox" class="custom-control-input" id="active" name="active" {{ $filter->active ? 'checked=""' : ''}}>
+                                <label class="custom-control-label font-weight-bold" for="active">Mark as Active</label>
+                            </div>
+                            <hr>
+                            <button type="submit" class="btn btn-success">Save</button>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+@endsection

resources/views/admin/asf/home.blade.php

@@ -0,0 +1,81 @@
+@extends('admin.partial.template-full')
+
+@section('section')
+</div><div class="header bg-primary pb-3 mt-n4">
+    <div class="container-fluid">
+        <div class="header-body">
+            <div class="row align-items-center py-4">
+                <div class="col-lg-6 col-7">
+                    <p class="display-1 text-white d-inline-block mb-0">Admin Shadow Filters</p>
+                    <p class="text-white mb-0">Manage shadow filters across Accounts, Hashtags, Feeds and Stories</p>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+    <div class="m-n2 m-lg-4">
+        <div class="container-fluid mt-4">
+            <div class="row mb-3 justify-content-between">
+                <div class="col-12 col-md-8">
+                    <ul class="nav nav-pills">
+                        <li class="nav-item">
+                            <a class="nav-link {{request()->has('filter') ? '':'active'}}" href="/i/admin/asf/home">Active Filters</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link {{request()->has('filter') && request()->filter == 'all' ? 'active':''}}" href="/i/admin/asf/home?filter=all">All</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link {{request()->has('filter') && request()->filter == 'inactive' ? 'active':''}}" href="/i/admin/asf/home?filter=inactive">Inactive</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link {{request()->has('new') ? 'active':''}}" href="/i/admin/asf/create">New</a>
+                        </li>
+                    </ul>
+                </div>
+
+                <div class="col-12 col-md-4">
+                    <form method="get">
+                        <input class="form-control" placeholder="Search by username" name="q" value="{{request()->has('q') ? request()->query('q') : ''}}" />
+                    </form>
+                </div>
+            </div>
+
+            <div class="table-responsive rounded">
+                <table class="table table-dark">
+                    <thead class="thead-dark">
+                        <tr>
+                            <th scope="col" class="cursor-pointer">ID</th>
+                            <th scope="col" class="cursor-pointer">Username</th>
+                            <th scope="col" class="cursor-pointer">Hide Feeds</th>
+                            <th scope="col" class="cursor-pointer">Active</th>
+                            <th scope="col" class="cursor-pointer">Created</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        @foreach($filters as $filter)
+                        <tr>
+                            <td><a href="/i/admin/asf/edit/{{$filter->id}}">{{ $filter->id }}</a></td>
+                            <td>
+                                <div class="d-flex align-items-center" style="gap: 1rem;">
+
+                                <img src="{{ $filter->account()['avatar'] }}" class="rounded-circle" width="30" height="30" onerror="this.src='/storage/avatars/default.jpg';this.onerror=null;" />
+                                <p class="font-weight-bold mb-0">
+                                    &commat;{{ $filter->account()['acct'] }}
+                                </p>
+                                </div>
+                            </td>
+                            <td>{{ $filter->hide_from_public_feeds ? '✅' : ''}}</td>
+                            <td>{{ $filter->active ? '✅' : ''}}</td>
+                            <td>{{ $filter->created_at->diffForHumans() }}</td>
+                        </tr>
+                        @endforeach
+                    </tbody>
+                </table>
+
+                <div class="d-flex mt-3">
+                    {{ $filters->links() }}
+                </div>
+            </div>
+        </div>
+    </div>
+@endsection

resources/views/auth/login.blade.php

@@ -74,7 +74,10 @@
                         </div>
 
                     </form>
-                    @if(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'))
+                    @if(
+                        (config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled')) ||
+                        (config('remote-auth.mastodon.ignore_closed_state') && config('remote-auth.mastodon.enabled'))
+                    )
                     <hr>
                     <form method="POST" action="/auth/raw/mastodon/start">
                         @csrf

resources/views/profile/embed.blade.php

@@ -16,8 +16,8 @@
     <meta name="medium" content="image">
     <meta name="theme-color" content="#10c5f8">
     <meta name="apple-mobile-web-app-capable" content="yes">
-    <link rel="shortcut icon" type="image/png" href="/img/favicon.png?v=2">
+    <link rel="shortcut icon" type="image/png" href="{{url('/img/favicon.png?v=2')}}">
-    <link rel="apple-touch-icon" type="image/png" href="/img/favicon.png?v=2">
+    <link rel="apple-touch-icon" type="image/png" href="{{url('/img/favicon.png?v=2')}}">
     <link href="{{ mix('css/app.css') }}" rel="stylesheet">
     <style type="text/css">
       body.embed-card {
@@ -73,7 +73,9 @@
 <script type="text/javascript" src="{{mix('js/manifest.js')}}"></script>
 <script type="text/javascript" src="{{mix('js/vendor.js')}}"></script>
 <script type="text/javascript" src="{{mix('js/app.js')}}"></script>
-  <script type="text/javascript">window.addEventListener("message",e=>{const t=e.data||{};window.parent&&"setHeight"===t.type&&window.parent.postMessage({type:"setHeight",id:t.id,height:document.getElementsByTagName("html")[0].scrollHeight},"*")});</script>
+<script type="text/javascript">
+  window.addEventListener("message", e=>{const t=e.data||{};});
+</script>
 <script type="text/javascript">document.querySelectorAll('.caption-container a').forEach(function(i) {i.setAttribute('target', '_blank');});</script>
 <script type="text/javascript">
     document.querySelectorAll('.prettyCount').forEach(function(i) {
@@ -84,13 +86,14 @@
  axios.get('/api/pixelfed/v1/accounts/{{$profile['id']}}/statuses', {
     params: {
        only_media: true,
-  			limit: 20
+       limit: 24
    }
 })
  .then(res => {
   let parent = $('.embed-row');
   res.data
   .filter(res => res.pf_type == 'photo')
+  .filter(res => !res.sensitive)
   .slice(0, 9)
   .forEach(post => {
      let el = `<div class="col-4 mt-2 px-0">
@@ -103,7 +106,13 @@
      </div>`;
      parent.append(el);
  })
-  	});
+})
+ .finally(() => {
+    window.parent.postMessage({type:"setHeight",id:0,height:document.getElementsByTagName("html")[0].scrollHeight},"*");
+    setTimeout(() => {
+        window.parent.postMessage({type:"setHeight",id:0,height:document.getElementsByTagName("html")[0].scrollHeight},"*");
+    }, 5000);
+})
 </script>
 </body>
 </html>

resources/views/settings/partial/sidebar.blade.php

@@ -72,6 +72,8 @@
 			@media only screen and (min-width: 768px) {
 				border-right: 1px solid #dee2e6 !important
 			}
+			height: 100%;
+			flex-grow: 1;
 		}
 	</style>
 	@endpush

resources/views/settings/privacy.blade.php

@@ -28,9 +28,17 @@
     <div class="form-check pb-3">
       <input class="form-check-input" type="checkbox" name="crawlable" id="crawlable" {{!$settings->crawlable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}>
       <label class="form-check-label font-weight-bold" for="crawlable">
-        {{__('Opt-out of search engine indexing')}}
+        {{__('Disable Search Engine indexing')}}
       </label>
-      <p class="text-muted small help-text">When your account is visible to search engines, your information can be crawled and stored by search engines.</p>
+      <p class="text-muted small help-text">When your account is visible to search engines, your information can be crawled and stored by search engines. {!! $settings->is_private ? '<strong>Not available when your account is private</strong>' : ''!!}</p>
+    </div>
+
+    <div class="form-check pb-3">
+      <input class="form-check-input" type="checkbox" name="indexable" id="indexable" {{$profile->indexable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}>
+      <label class="form-check-label font-weight-bold" for="indexable">
+        {{__('Include public posts in search results')}}
+      </label>
+        <p class="text-muted small help-text">Your public posts may appear in search results on Pixelfed and Mastodon. People who have interacted with your posts may be able to search them regardless. {!! $settings->is_private ? '<strong>Not available when your account is private</strong>' : ''!!}</p>
     </div>
 
 
@@ -39,7 +47,7 @@
       <label class="form-check-label font-weight-bold" for="is_suggestable">
         {{__('Show on Directory')}}
       </label>
-      <p class="text-muted small help-text">When this option is enabled, your profile is included in the Directory. Only public profiles are eligible.</p>
+      <p class="text-muted small help-text">When this option is enabled, your profile is included in the Directory. Only public profiles are eligible. {!! $settings->is_private ? '<strong>Not available when your account is private</strong>' : ''!!}</p>
     </div>
 
     <div class="form-check pb-3">

routes/api.php

@@ -316,6 +316,7 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
                 Route::post('seen', 'Stories\StoryApiV1Controller@viewed')->middleware($middleware);
                 Route::post('self-expire/{id}', 'Stories\StoryApiV1Controller@delete')->middleware($middleware);
                 Route::post('comment', 'Stories\StoryApiV1Controller@comment')->middleware($middleware);
+                Route::get('viewers', 'Stories\StoryApiV1Controller@viewers')->middleware($middleware);
             });
         });
     });

routes/web.php

@@ -96,6 +96,13 @@ Route::domain(config('pixelfed.domain.admin'))->prefix('i/admin')->group(functio
 
 	Route::get('autospam/home', 'AdminController@autospamHome')->name('admin.autospam');
 
+    Route::redirect('asf/', 'asf/home');
+    Route::get('asf/home', 'AdminShadowFilterController@home');
+    Route::get('asf/create', 'AdminShadowFilterController@create');
+    Route::get('asf/edit/{id}', 'AdminShadowFilterController@edit');
+    Route::post('asf/edit/{id}', 'AdminShadowFilterController@storeEdit');
+    Route::post('asf/create', 'AdminShadowFilterController@store');
+
 	Route::prefix('api')->group(function() {
 		Route::get('stats', 'AdminController@getStats');
 		Route::get('accounts', 'AdminController@getAccounts');

tests/Unit/ActivityPubTagObjectTest.php

@@ -0,0 +1,133 @@
+<?php
+
+namespace Tests\Unit;
+
+use PHPUnit\Framework\TestCase;
+
+class ActivityPubTagObjectTest extends TestCase
+{
+    /**
+     * A basic unit test example.
+     */
+    public function test_gotosocial(): void
+    {
+        $res = [
+            "tag" => [
+                "href" => "https://gotosocial.example.org/users/GotosocialUser",
+                "name" => "@GotosocialUser@gotosocial.example.org",
+                "type" => "Mention"
+            ]
+        ];
+
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
+
+        $tags = collect($res['tag'])
+        ->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Mention' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        });
+        $this->assertTrue($tags->count() === 1);
+    }
+
+    public function test_pixelfed_hashtags(): void
+    {
+        $res = [
+            "tag" => [
+                [
+                    "type" => "Mention",
+                    "href" => "https://pixelfed.social/dansup",
+                    "name" => "@dansup@pixelfed.social"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed",
+                    "name" => "#dogsOfPixelFed"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/doggo",
+                    "name" => "#doggo"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/dog",
+                    "name" => "#dog"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/drake",
+                    "name" => "#drake"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/blacklab",
+                    "name" => "#blacklab"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/iconic",
+                    "name" => "#Iconic"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/majestic",
+                    "name" => "#majestic"
+                ]
+            ]
+        ];
+
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
+
+        $tags = collect($res['tag'])
+        ->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Hashtag' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        });
+        $this->assertTrue($tags->count() === 7);
+    }
+
+
+    public function test_pixelfed_mentions(): void
+    {
+        $res = [
+            "tag" => [
+                [
+                    "type" => "Mention",
+                    "href" => "https://pixelfed.social/dansup",
+                    "name" => "@dansup@pixelfed.social"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed",
+                    "name" => "#dogsOfPixelFed"
+                ],
+                [
+                    "type" => "Hashtag",
+                    "href" => "https://pixelfed.social/discover/tags/doggo",
+                    "name" => "#doggo"
+                ],
+            ]
+        ];
+
+        if(isset($res['tag']['type'], $res['tag']['name'])) {
+            $res['tag'] = [$res['tag']];
+        }
+
+        $tags = collect($res['tag'])
+        ->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Mention' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        });
+        $this->assertTrue($tags->count() === 1);
+    }
+}

tests/Unit/Lexer/UsernameTest.php

@@ -175,4 +175,67 @@ class UsernameTest extends TestCase
         $this->assertEquals($expectedEntity, $entities);
     }
 
+    /** @test * */
+    public function germanUmlatsAutolink()
+    {
+        $mentions = "@März and @königin and @Glück";
+        $autolink = Autolink::create()->autolink($mentions);
+
+        $expectedAutolink = '<a class="u-url mention" href="https://pixelfed.dev/März" rel="external nofollow noopener" target="_blank">@März</a> and <a class="u-url mention" href="https://pixelfed.dev/königin" rel="external nofollow noopener" target="_blank">@königin</a> and <a class="u-url mention" href="https://pixelfed.dev/Glück" rel="external nofollow noopener" target="_blank">@Glück</a>';
+        $this->assertEquals($expectedAutolink, $autolink);
+    }
+
+    /** @test * */
+    public function germanUmlatsExtractor()
+    {
+        $mentions = "@März and @königin and @Glück";
+        $entities = Extractor::create()->extract($mentions);
+
+        $expectedEntity = [
+            "hashtags" => [],
+            "urls" => [],
+            "mentions" => [
+              "märz",
+              "königin",
+              "glück",
+            ],
+            "replyto" => null,
+            "hashtags_with_indices" => [],
+            "urls_with_indices" => [],
+            "mentions_with_indices" => [
+              [
+                "screen_name" => "März",
+                "indices" => [
+                  0,
+                  5,
+                ],
+              ],
+              [
+                "screen_name" => "königin",
+                "indices" => [
+                  10,
+                  18,
+                ],
+              ],
+              [
+                "screen_name" => "Glück",
+                "indices" => [
+                  23,
+                  29,
+                ],
+              ],
+            ],
+        ];
+        $this->assertEquals($expectedEntity, $entities);
+    }
+
+    /** @test * */
+    public function germanUmlatsWebfingerAutolink()
+    {
+        $mentions = "hello @märz@example.org!";
+        $autolink = Autolink::create()->autolink($mentions);
+
+        $expectedAutolink = 'hello <a class="u-url list-slug" href="https://pixelfed.dev/@märz@example.org" rel="external nofollow noopener" target="_blank">@märz@example.org</a>!';
+        $this->assertEquals($expectedAutolink, $autolink);
+    }
 }