Refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances

app/Console/Commands/InstanceUpdateTotalLocalPosts.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\ConfigCacheService;
+use Cache;
+use DB;
+use Illuminate\Console\Command;
+use Storage;
+
+class InstanceUpdateTotalLocalPosts extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:instance-update-total-local-posts';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update the total number of local statuses/post count';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $cached = $this->checkForCache();
+        if (! $cached) {
+            $this->initCache();
+
+            return;
+        }
+        $cache = $this->getCached();
+        if (! $cache || ! isset($cache['count'])) {
+            $this->error('Problem fetching cache');
+
+            return;
+        }
+        $this->updateAndCache();
+        Cache::forget('api:nodeinfo');
+
+    }
+
+    protected function checkForCache()
+    {
+        return Storage::exists('total_local_posts.json');
+    }
+
+    protected function initCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+    }
+
+    protected function getCached()
+    {
+        return Storage::json('total_local_posts.json');
+    }
+
+    protected function updateAndCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+
+    }
+}

app/Console/Kernel.php

@@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
         $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
         $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
         $schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
+        $schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
     }
 
     /**

app/Http/Controllers/PixelfedDirectoryController.php

@@ -4,13 +4,13 @@ namespace App\Http\Controllers;
 
 use App\Models\ConfigCache;
 use App\Services\AccountService;
+use App\Services\InstanceService;
 use App\Services\StatusService;
+use App\User;
+use Cache;
 use Illuminate\Http\Request;
 use Illuminate\Support\Str;
-use Cache;
 use Storage;
-use App\Status;
-use App\User;
 
 class PixelfedDirectoryController extends Controller
 {
@@ -140,9 +140,7 @@ class PixelfedDirectoryController extends Controller
             'stories' => (bool) config_cache('instance.stories.enabled'),
         ];
 
-        $statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-            return Status::whereLocal(true)->count();
-        });
+        $statusesCount = InstanceService::totalLocalStatuses();
         $usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
             return User::count();
         });

app/Services/ConfigCacheService.php

@@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
 class ConfigCacheService
 {
     const CACHE_KEY = 'config_cache:_v0-key:';
+
     const PROTECTED_KEYS = [
         'filesystems.disks.s3.key',
         'filesystems.disks.s3.secret',
@@ -133,6 +134,8 @@ class ConfigCacheService
                     'filesystems.disks.spaces.url',
                     'filesystems.disks.spaces.endpoint',
                     'filesystems.disks.spaces.use_path_style_endpoint',
+
+                    'instance.stats.total_local_posts',
                     // 'system.user_mode'
                 ];
 

app/Services/InstanceService.php

@@ -18,6 +18,8 @@ class InstanceService
 
     const CACHE_KEY_STATS = 'pf:services:instances:stats';
 
+    const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
+
     const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
 
     const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
@@ -96,6 +98,11 @@ class InstanceService
         return true;
     }
 
+    public static function totalLocalStatuses()
+    {
+        return config_cache('instance.stats.total_local_posts');
+    }
+
     public static function headerBlurhash()
     {
         return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {

app/Services/LandingService.php

@@ -2,7 +2,6 @@
 
 namespace App\Services;
 
-use App\Status;
 use App\User;
 use App\Util\Site\Nodeinfo;
 use Illuminate\Support\Facades\Cache;
@@ -18,9 +17,7 @@ class LandingService
             return User::count();
         });
 
-        $postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
-            return Status::whereLocal(true)->count();
-        });
+        $postCount = InstanceService::totalLocalStatuses();
 
         $contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
             if (config_cache('instance.admin.pid')) {

app/Services/StatusService.php

@@ -2,15 +2,11 @@
 
 namespace App\Services;
 
-use Illuminate\Support\Facades\Cache;
-use Illuminate\Support\Facades\Redis;
-use DB;
 use App\Status;
 use App\Transformer\Api\StatusStatelessTransformer;
-use App\Transformer\Api\StatusTransformer;
+use Illuminate\Support\Facades\Cache;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 
 class StatusService
 {
@@ -19,6 +15,7 @@ class StatusService
     public static function key($id, $publicOnly = true)
     {
         $p = $publicOnly ? 'pub:' : 'all:';
+
         return self::CACHE_KEY.$p.$id;
     }
 
@@ -41,12 +38,14 @@ class StatusService
             if (isset($res['_pid'])) {
                 unset($res['account']);
             }
+
             return $res;
         });
         if ($res && isset($res['_pid'])) {
             $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
             unset($res['_pid']);
         }
+
         return $res;
     }
 
@@ -117,14 +116,14 @@ class StatusService
             return [
                 'liked' => false,
                 'shared' => false,
-                'bookmarked' => false
+                'bookmarked' => false,
             ];
         }
 
         return [
             'liked' => LikeService::liked($pid, $id),
             'shared' => self::isShared($id, $pid),
-            'bookmarked' => self::isBookmarked($id, $pid)
+            'bookmarked' => self::isBookmarked($id, $pid),
         ];
     }
 
@@ -135,6 +134,7 @@ class StatusService
             return $res;
         }
         $res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
+
         return $res;
     }
 
@@ -149,6 +149,7 @@ class StatusService
         $fractal = new Fractal\Manager();
         $fractal->setSerializer(new ArraySerializer());
         $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
+
         return $fractal->createData($resource)->toArray();
     }
 
@@ -167,6 +168,7 @@ class StatusService
         }
 
         Cache::forget(self::key($id, false));
+
         return Cache::forget(self::key($id));
     }
 
@@ -194,8 +196,6 @@ class StatusService
 
     public static function totalLocalStatuses()
     {
-        return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
-            return Status::whereNull('url')->count();
-        });
+        return InstanceService::totalLocalStatuses();
     }
 }

app/Util/Site/Nodeinfo.php

@@ -2,12 +2,9 @@
 
 namespace App\Util\Site;
 
-use Illuminate\Support\Facades\Cache;
-use App\Like;
-use App\Profile;
-use App\Status;
+use App\Services\InstanceService;
 use App\User;
-use Illuminate\Support\Str;
+use Illuminate\Support\Facades\Cache;
 
 class Nodeinfo
 {
@@ -21,9 +18,7 @@ class Nodeinfo
                 return User::count();
             });
 
-            $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-                return Status::whereLocal(true)->count();
-            });
+            $statuses = InstanceService::totalLocalStatuses();
 
             $features = ['features' => \App\Util\Site\Config::get()['features']];
 
@@ -34,7 +29,7 @@ class Nodeinfo
                         'homepage' => 'https://pixelfed.org',
                         'repo' => 'https://github.com/pixelfed/pixelfed',
                     ],
-                    'config' => $features
+                    'config' => $features,
                 ],
                 'protocols' => [
                     'activitypub',
@@ -60,6 +55,7 @@ class Nodeinfo
             ];
         });
         $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
+
         return $res;
     }
 

database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+use App\Services\ConfigCacheService;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+        Cache::forget('api:nodeinfo');
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        //
+    }
+};