From 4f2b8ed20ad9cbb70bd07c720046e67543f857b7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 19 Jun 2024 03:02:00 -0600 Subject: [PATCH] Refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances --- .../InstanceUpdateTotalLocalPosts.php | 79 +++++++++++++++++++ app/Console/Kernel.php | 1 + .../PixelfedDirectoryController.php | 12 ++- app/Services/ConfigCacheService.php | 13 +-- app/Services/InstanceService.php | 7 ++ app/Services/LandingService.php | 5 +- app/Services/StatusService.php | 54 ++++++------- app/Util/Site/Nodeinfo.php | 52 ++++++------ ..._add_total_local_posts_to_config_cache.php | 34 ++++++++ 9 files changed, 186 insertions(+), 71 deletions(-) create mode 100644 app/Console/Commands/InstanceUpdateTotalLocalPosts.php create mode 100644 database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php diff --git a/app/Console/Commands/InstanceUpdateTotalLocalPosts.php b/app/Console/Commands/InstanceUpdateTotalLocalPosts.php new file mode 100644 index 000000000..d44236a51 --- /dev/null +++ b/app/Console/Commands/InstanceUpdateTotalLocalPosts.php @@ -0,0 +1,79 @@ +checkForCache(); + if (! $cached) { + $this->initCache(); + + return; + } + $cache = $this->getCached(); + if (! $cache || ! isset($cache['count'])) { + $this->error('Problem fetching cache'); + + return; + } + $this->updateAndCache(); + Cache::forget('api:nodeinfo'); + + } + + protected function checkForCache() + { + return Storage::exists('total_local_posts.json'); + } + + protected function initCache() + { + $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count(); + $res = [ + 'count' => $count, + ]; + Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT)); + ConfigCacheService::put('instance.stats.total_local_posts', $res['count']); + } + + protected function getCached() + { + return Storage::json('total_local_posts.json'); + } + + protected function updateAndCache() + { + $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count(); + $res = [ + 'count' => $count, + ]; + Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT)); + ConfigCacheService::put('instance.stats.total_local_posts', $res['count']); + + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 37f9d20b3..d5f6962f4 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer(); $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer(); $schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer(); + $schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer(); } /** diff --git a/app/Http/Controllers/PixelfedDirectoryController.php b/app/Http/Controllers/PixelfedDirectoryController.php index 0d2113a04..0477c5170 100644 --- a/app/Http/Controllers/PixelfedDirectoryController.php +++ b/app/Http/Controllers/PixelfedDirectoryController.php @@ -4,13 +4,13 @@ namespace App\Http\Controllers; use App\Models\ConfigCache; use App\Services\AccountService; +use App\Services\InstanceService; use App\Services\StatusService; +use App\User; +use Cache; use Illuminate\Http\Request; use Illuminate\Support\Str; -use Cache; use Storage; -use App\Status; -use App\User; class PixelfedDirectoryController extends Controller { @@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller 'stories' => (bool) config_cache('instance.stories.enabled'), ]; - $statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() { - return Status::whereLocal(true)->count(); - }); - $usersCount = Cache::remember('api:nodeinfo:users', 43200, function() { + $statusesCount = InstanceService::totalLocalStatuses(); + $usersCount = Cache::remember('api:nodeinfo:users', 43200, function () { return User::count(); }); $res['stats'] = [ diff --git a/app/Services/ConfigCacheService.php b/app/Services/ConfigCacheService.php index b18c02e36..4f2b006cc 100644 --- a/app/Services/ConfigCacheService.php +++ b/app/Services/ConfigCacheService.php @@ -9,6 +9,7 @@ use Illuminate\Database\QueryException; class ConfigCacheService { const CACHE_KEY = 'config_cache:_v0-key:'; + const PROTECTED_KEYS = [ 'filesystems.disks.s3.key', 'filesystems.disks.s3.secret', @@ -133,6 +134,8 @@ class ConfigCacheService 'filesystems.disks.spaces.url', 'filesystems.disks.spaces.endpoint', 'filesystems.disks.spaces.use_path_style_endpoint', + + 'instance.stats.total_local_posts', // 'system.user_mode' ]; @@ -146,7 +149,7 @@ class ConfigCacheService $protect = false; $protected = null; - if(in_array($key, self::PROTECTED_KEYS)) { + if (in_array($key, self::PROTECTED_KEYS)) { $protect = true; } @@ -154,7 +157,7 @@ class ConfigCacheService $c = ConfigCacheModel::where('k', $key)->first(); if ($c) { - if($protect) { + if ($protect) { return decrypt($c->v) ?? config($key); } else { return $c->v ?? config($key); @@ -165,7 +168,7 @@ class ConfigCacheService return; } - if($protect && $v) { + if ($protect && $v) { $protected = encrypt($v); } @@ -176,7 +179,7 @@ class ConfigCacheService return $v; }); - } catch (Exception | QueryException $e) { + } catch (Exception|QueryException $e) { return config($key); } } @@ -187,7 +190,7 @@ class ConfigCacheService $protect = false; $protected = null; - if(in_array($key, self::PROTECTED_KEYS)) { + if (in_array($key, self::PROTECTED_KEYS)) { $protect = true; $protected = encrypt($val); } diff --git a/app/Services/InstanceService.php b/app/Services/InstanceService.php index c07e17521..0a6255ad2 100644 --- a/app/Services/InstanceService.php +++ b/app/Services/InstanceService.php @@ -18,6 +18,8 @@ class InstanceService const CACHE_KEY_STATS = 'pf:services:instances:stats'; + const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts'; + const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1'; const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0'; @@ -96,6 +98,11 @@ class InstanceService return true; } + public static function totalLocalStatuses() + { + return config_cache('instance.stats.total_local_posts'); + } + public static function headerBlurhash() { return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () { diff --git a/app/Services/LandingService.php b/app/Services/LandingService.php index f51822df2..d6180771d 100644 --- a/app/Services/LandingService.php +++ b/app/Services/LandingService.php @@ -2,7 +2,6 @@ namespace App\Services; -use App\Status; use App\User; use App\Util\Site\Nodeinfo; use Illuminate\Support\Facades\Cache; @@ -18,9 +17,7 @@ class LandingService return User::count(); }); - $postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () { - return Status::whereLocal(true)->count(); - }); + $postCount = InstanceService::totalLocalStatuses(); $contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () { if (config_cache('instance.admin.pid')) { diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index 44f33fcb9..d73f6c018 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -2,15 +2,11 @@ namespace App\Services; -use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Redis; -use DB; use App\Status; use App\Transformer\Api\StatusStatelessTransformer; -use App\Transformer\Api\StatusTransformer; +use Illuminate\Support\Facades\Cache; use League\Fractal; use League\Fractal\Serializer\ArraySerializer; -use League\Fractal\Pagination\IlluminatePaginatorAdapter; class StatusService { @@ -19,18 +15,19 @@ class StatusService public static function key($id, $publicOnly = true) { $p = $publicOnly ? 'pub:' : 'all:'; - return self::CACHE_KEY . $p . $id; + + return self::CACHE_KEY.$p.$id; } public static function get($id, $publicOnly = true, $mastodonMode = false) { - $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) { - if($publicOnly) { + $res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) { + if ($publicOnly) { $status = Status::whereScope('public')->find($id); } else { $status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id); } - if(!$status) { + if (! $status) { return null; } $fractal = new Fractal\Manager(); @@ -38,32 +35,34 @@ class StatusService $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer()); $res = $fractal->createData($resource)->toArray(); $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null; - if(isset($res['_pid'])) { + if (isset($res['_pid'])) { unset($res['account']); } + return $res; }); - if($res && isset($res['_pid'])) { + if ($res && isset($res['_pid'])) { $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true); unset($res['_pid']); } + return $res; } public static function getMastodon($id, $publicOnly = true) { $status = self::get($id, $publicOnly, true); - if(!$status) { + if (! $status) { return null; } - if(!isset($status['account'])) { + if (! isset($status['account'])) { return null; } $status['replies_count'] = $status['reply_count']; - if(config('exp.emc') == false) { + if (config('exp.emc') == false) { return $status; } @@ -113,28 +112,29 @@ class StatusService { $status = self::get($id, false); - if(!$status) { + if (! $status) { return [ 'liked' => false, 'shared' => false, - 'bookmarked' => false + 'bookmarked' => false, ]; } return [ 'liked' => LikeService::liked($pid, $id), 'shared' => self::isShared($id, $pid), - 'bookmarked' => self::isBookmarked($id, $pid) + 'bookmarked' => self::isBookmarked($id, $pid), ]; } public static function getFull($id, $pid, $publicOnly = true) { $res = self::get($id, $publicOnly); - if(!$res || !isset($res['account']) || !isset($res['account']['id'])) { + if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) { return $res; } $res['relationship'] = RelationshipService::get($pid, $res['account']['id']); + return $res; } @@ -142,31 +142,33 @@ class StatusService { $status = Status::whereScope('direct')->find($id); - if(!$status) { + if (! $status) { return null; } $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer()); + return $fractal->createData($resource)->toArray(); } public static function del($id, $purge = false) { - if($purge) { + if ($purge) { $status = self::get($id); - if($status && isset($status['account']) && isset($status['account']['id'])) { - Cache::forget('profile:embed:' . $status['account']['id']); + if ($status && isset($status['account']) && isset($status['account']['id'])) { + Cache::forget('profile:embed:'.$status['account']['id']); } - Cache::forget('status:transformer:media:attachments:' . $id); + Cache::forget('status:transformer:media:attachments:'.$id); MediaService::del($id); - Cache::forget('pf:services:sh:id:' . $id); + Cache::forget('pf:services:sh:id:'.$id); PublicTimelineService::rem($id); NetworkTimelineService::rem($id); } Cache::forget(self::key($id, false)); + return Cache::forget(self::key($id)); } @@ -194,8 +196,6 @@ class StatusService public static function totalLocalStatuses() { - return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() { - return Status::whereNull('url')->count(); - }); + return InstanceService::totalLocalStatuses(); } } diff --git a/app/Util/Site/Nodeinfo.php b/app/Util/Site/Nodeinfo.php index 0458299c5..9c0031ef4 100644 --- a/app/Util/Site/Nodeinfo.php +++ b/app/Util/Site/Nodeinfo.php @@ -2,12 +2,9 @@ namespace App\Util\Site; -use Illuminate\Support\Facades\Cache; -use App\Like; -use App\Profile; -use App\Status; +use App\Services\InstanceService; use App\User; -use Illuminate\Support\Str; +use Illuminate\Support\Facades\Cache; class Nodeinfo { @@ -17,49 +14,48 @@ class Nodeinfo $activeHalfYear = self::activeUsersHalfYear(); $activeMonth = self::activeUsersMonthly(); - $users = Cache::remember('api:nodeinfo:users', 43200, function() { + $users = Cache::remember('api:nodeinfo:users', 43200, function () { return User::count(); }); - $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() { - return Status::whereLocal(true)->count(); - }); + $statuses = InstanceService::totalLocalStatuses(); - $features = [ 'features' => \App\Util\Site\Config::get()['features'] ]; + $features = ['features' => \App\Util\Site\Config::get()['features']]; return [ 'metadata' => [ 'nodeName' => config_cache('app.name'), 'software' => [ - 'homepage' => 'https://pixelfed.org', - 'repo' => 'https://github.com/pixelfed/pixelfed', + 'homepage' => 'https://pixelfed.org', + 'repo' => 'https://github.com/pixelfed/pixelfed', ], - 'config' => $features + 'config' => $features, ], - 'protocols' => [ + 'protocols' => [ 'activitypub', ], 'services' => [ - 'inbound' => [], + 'inbound' => [], 'outbound' => [], ], 'software' => [ - 'name' => 'pixelfed', - 'version' => config('pixelfed.version'), + 'name' => 'pixelfed', + 'version' => config('pixelfed.version'), ], 'usage' => [ - 'localPosts' => (int) $statuses, + 'localPosts' => (int) $statuses, 'localComments' => 0, - 'users' => [ - 'total' => (int) $users, + 'users' => [ + 'total' => (int) $users, 'activeHalfyear' => (int) $activeHalfYear, - 'activeMonth' => (int) $activeMonth, + 'activeMonth' => (int) $activeMonth, ], ], 'version' => '2.0', ]; }); $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration'); + return $res; } @@ -69,7 +65,7 @@ class Nodeinfo 'links' => [ [ 'href' => config('pixelfed.nodeinfo.url'), - 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', + 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', ], ], ]; @@ -77,18 +73,18 @@ class Nodeinfo public static function activeUsersMonthly() { - return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() { + return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () { return User::withTrashed() - ->select('last_active_at, updated_at') - ->where('updated_at', '>', now()->subWeeks(5)) - ->orWhere('last_active_at', '>', now()->subWeeks(5)) - ->count(); + ->select('last_active_at, updated_at') + ->where('updated_at', '>', now()->subWeeks(5)) + ->orWhere('last_active_at', '>', now()->subWeeks(5)) + ->count(); }); } public static function activeUsersHalfYear() { - return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() { + return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () { return User::withTrashed() ->select('last_active_at, updated_at') ->where('last_active_at', '>', now()->subMonths(6)) diff --git a/database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php b/database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php new file mode 100644 index 000000000..35f00f60a --- /dev/null +++ b/database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php @@ -0,0 +1,34 @@ +whereNull(['url', 'deleted_at'])->count(); + $res = [ + 'count' => $count + ]; + Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT)); + ConfigCacheService::put('instance.stats.total_local_posts', $res['count']); + Cache::forget('api:nodeinfo'); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +};