Merge pull request #5184 from pixelfed/staging

Staging
2024-11-17 20:11:27 +00:00 · 2024-06-19 03:10:29 -06:00 · 2024-06-19 03:10:29 -06:00 · d13e9e8e5d
commit d13e9e8e5d
parent 53faeefd40 62ee6c78c8
10 changed files with 189 additions and 71 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -28,6 +28,9 @@
 - Update StatusController, cache AP object ([a75b89b2](https://github.com/pixelfed/pixelfed/commit/a75b89b2))
 - Update status embed, add support for album carousels ([f4898db9](https://github.com/pixelfed/pixelfed/commit/f4898db9))
 - Update profile embeds, add support for albums ([4fd156c4](https://github.com/pixelfed/pixelfed/commit/4fd156c4))
+- Update DirectMessageController, add timestamps to threads ([b24d2554](https://github.com/pixelfed/pixelfed/commit/b24d2554))
+- Update DirectMessageController, add carousel entity to threads ([96f24f33](https://github.com/pixelfed/pixelfed/commit/96f24f33))
+- Update and refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances ([4f2b8ed2](https://github.com/pixelfed/pixelfed/commit/4f2b8ed2))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))

--- a/app/Console/Commands/InstanceUpdateTotalLocalPosts.php
+++ b/app/Console/Commands/InstanceUpdateTotalLocalPosts.php
@ -0,0 +1,79 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\ConfigCacheService;
+use Cache;
+use DB;
+use Illuminate\Console\Command;
+use Storage;
+
+class InstanceUpdateTotalLocalPosts extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:instance-update-total-local-posts';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update the total number of local statuses/post count';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $cached = $this->checkForCache();
+        if (! $cached) {
+            $this->initCache();
+
+            return;
+        }
+        $cache = $this->getCached();
+        if (! $cache || ! isset($cache['count'])) {
+            $this->error('Problem fetching cache');
+
+            return;
+        }
+        $this->updateAndCache();
+        Cache::forget('api:nodeinfo');
+
+    }
+
+    protected function checkForCache()
+    {
+        return Storage::exists('total_local_posts.json');
+    }
+
+    protected function initCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+    }
+
+    protected function getCached()
+    {
+        return Storage::json('total_local_posts.json');
+    }
+
+    protected function updateAndCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+
+    }
+}
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
        $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
        $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
        $schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
+        $schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
    }

    /**
--- a/app/Http/Controllers/PixelfedDirectoryController.php
+++ b/app/Http/Controllers/PixelfedDirectoryController.php
@ -4,13 +4,13 @@ namespace App\Http\Controllers;

 use App\Models\ConfigCache;
 use App\Services\AccountService;
+use App\Services\InstanceService;
 use App\Services\StatusService;
+use App\User;
+use Cache;
 use Illuminate\Http\Request;
 use Illuminate\Support\Str;
-use Cache;
 use Storage;
-use App\Status;
-use App\User;

 class PixelfedDirectoryController extends Controller
 {
@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller
            'stories' => (bool) config_cache('instance.stories.enabled'),
        ];

-        $statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-            return Status::whereLocal(true)->count();
-        });
-        $usersCount = Cache::remember('api:nodeinfo:users', 43200, function() {
+        $statusesCount = InstanceService::totalLocalStatuses();
+        $usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
            return User::count();
        });
        $res['stats'] = [
--- a/app/Services/ConfigCacheService.php
+++ b/app/Services/ConfigCacheService.php
@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
 class ConfigCacheService
 {
    const CACHE_KEY = 'config_cache:_v0-key:';
+
    const PROTECTED_KEYS = [
        'filesystems.disks.s3.key',
        'filesystems.disks.s3.secret',
@ -133,6 +134,8 @@ class ConfigCacheService
                    'filesystems.disks.spaces.url',
                    'filesystems.disks.spaces.endpoint',
                    'filesystems.disks.spaces.use_path_style_endpoint',
+
+                    'instance.stats.total_local_posts',
                    // 'system.user_mode'
                ];

@ -146,7 +149,7 @@ class ConfigCacheService

                $protect = false;
                $protected = null;
-                if(in_array($key, self::PROTECTED_KEYS)) {
+                if (in_array($key, self::PROTECTED_KEYS)) {
                    $protect = true;
                }

@ -154,7 +157,7 @@ class ConfigCacheService
                $c = ConfigCacheModel::where('k', $key)->first();

                if ($c) {
-                    if($protect) {
+                    if ($protect) {
                        return decrypt($c->v) ?? config($key);
                    } else {
                        return $c->v ?? config($key);
@ -165,7 +168,7 @@ class ConfigCacheService
                    return;
                }

-                if($protect && $v) {
+                if ($protect && $v) {
                    $protected = encrypt($v);
                }

@ -176,7 +179,7 @@ class ConfigCacheService

                return $v;
            });
-        } catch (Exception | QueryException $e) {
+        } catch (Exception|QueryException $e) {
            return config($key);
        }
    }
@ -187,7 +190,7 @@ class ConfigCacheService

        $protect = false;
        $protected = null;
-        if(in_array($key, self::PROTECTED_KEYS)) {
+        if (in_array($key, self::PROTECTED_KEYS)) {
            $protect = true;
            $protected = encrypt($val);
        }
--- a/app/Services/InstanceService.php
+++ b/app/Services/InstanceService.php
@ -18,6 +18,8 @@ class InstanceService

    const CACHE_KEY_STATS = 'pf:services:instances:stats';

+    const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
+
    const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';

    const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
@ -96,6 +98,11 @@ class InstanceService
        return true;
    }

+    public static function totalLocalStatuses()
+    {
+        return config_cache('instance.stats.total_local_posts');
+    }
+
    public static function headerBlurhash()
    {
        return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {
--- a/app/Services/LandingService.php
+++ b/app/Services/LandingService.php
@ -2,7 +2,6 @@

 namespace App\Services;

-use App\Status;
 use App\User;
 use App\Util\Site\Nodeinfo;
 use Illuminate\Support\Facades\Cache;
@ -18,9 +17,7 @@ class LandingService
            return User::count();
        });

-        $postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
-            return Status::whereLocal(true)->count();
-        });
+        $postCount = InstanceService::totalLocalStatuses();

        $contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
            if (config_cache('instance.admin.pid')) {
--- a/app/Services/StatusService.php
+++ b/app/Services/StatusService.php
@ -2,15 +2,11 @@

 namespace App\Services;

-use Illuminate\Support\Facades\Cache;
-use Illuminate\Support\Facades\Redis;
-use DB;
 use App\Status;
 use App\Transformer\Api\StatusStatelessTransformer;
-use App\Transformer\Api\StatusTransformer;
+use Illuminate\Support\Facades\Cache;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use League\Fractal\Pagination\IlluminatePaginatorAdapter;

 class StatusService
 {
@ -19,18 +15,19 @@ class StatusService
    public static function key($id, $publicOnly = true)
    {
        $p = $publicOnly ? 'pub:' : 'all:';
-        return self::CACHE_KEY . $p . $id;
+
+        return self::CACHE_KEY.$p.$id;
    }

    public static function get($id, $publicOnly = true, $mastodonMode = false)
    {
-        $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
-            if($publicOnly) {
+        $res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) {
+            if ($publicOnly) {
                $status = Status::whereScope('public')->find($id);
            } else {
                $status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id);
            }
-            if(!$status) {
+            if (! $status) {
                return null;
            }
            $fractal = new Fractal\Manager();
@ -38,32 +35,34 @@ class StatusService
            $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
            $res = $fractal->createData($resource)->toArray();
            $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
-            if(isset($res['_pid'])) {
+            if (isset($res['_pid'])) {
                unset($res['account']);
            }
+
            return $res;
        });
-        if($res && isset($res['_pid'])) {
+        if ($res && isset($res['_pid'])) {
            $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
            unset($res['_pid']);
        }
+
        return $res;
    }

    public static function getMastodon($id, $publicOnly = true)
    {
        $status = self::get($id, $publicOnly, true);
-        if(!$status) {
+        if (! $status) {
            return null;
        }

-        if(!isset($status['account'])) {
+        if (! isset($status['account'])) {
            return null;
        }

        $status['replies_count'] = $status['reply_count'];

-        if(config('exp.emc') == false) {
+        if (config('exp.emc') == false) {
            return $status;
        }

@ -113,28 +112,29 @@ class StatusService
    {
        $status = self::get($id, false);

-        if(!$status) {
+        if (! $status) {
            return [
                'liked' => false,
                'shared' => false,
-                'bookmarked' => false
+                'bookmarked' => false,
            ];
        }

        return [
            'liked' => LikeService::liked($pid, $id),
            'shared' => self::isShared($id, $pid),
-            'bookmarked' => self::isBookmarked($id, $pid)
+            'bookmarked' => self::isBookmarked($id, $pid),
        ];
    }

    public static function getFull($id, $pid, $publicOnly = true)
    {
        $res = self::get($id, $publicOnly);
-        if(!$res || !isset($res['account']) || !isset($res['account']['id'])) {
+        if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) {
            return $res;
        }
        $res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
+
        return $res;
    }

@ -142,31 +142,33 @@ class StatusService
    {
        $status = Status::whereScope('direct')->find($id);

-        if(!$status) {
+        if (! $status) {
            return null;
        }

        $fractal = new Fractal\Manager();
        $fractal->setSerializer(new ArraySerializer());
        $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
+
        return $fractal->createData($resource)->toArray();
    }

    public static function del($id, $purge = false)
    {
-        if($purge) {
+        if ($purge) {
            $status = self::get($id);
-            if($status && isset($status['account']) && isset($status['account']['id'])) {
-                Cache::forget('profile:embed:' . $status['account']['id']);
+            if ($status && isset($status['account']) && isset($status['account']['id'])) {
+                Cache::forget('profile:embed:'.$status['account']['id']);
            }
-            Cache::forget('status:transformer:media:attachments:' . $id);
+            Cache::forget('status:transformer:media:attachments:'.$id);
            MediaService::del($id);
-            Cache::forget('pf:services:sh:id:' . $id);
+            Cache::forget('pf:services:sh:id:'.$id);
            PublicTimelineService::rem($id);
            NetworkTimelineService::rem($id);
        }

        Cache::forget(self::key($id, false));
+
        return Cache::forget(self::key($id));
    }

@ -194,8 +196,6 @@ class StatusService

    public static function totalLocalStatuses()
    {
-        return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
-            return Status::whereNull('url')->count();
-        });
+        return InstanceService::totalLocalStatuses();
    }
 }
--- a/app/Util/Site/Nodeinfo.php
+++ b/app/Util/Site/Nodeinfo.php
@ -2,12 +2,9 @@

 namespace App\Util\Site;

-use Illuminate\Support\Facades\Cache;
-use App\Like;
-use App\Profile;
-use App\Status;
+use App\Services\InstanceService;
 use App\User;
-use Illuminate\Support\Str;
+use Illuminate\Support\Facades\Cache;

 class Nodeinfo
 {
@ -17,49 +14,48 @@ class Nodeinfo
            $activeHalfYear = self::activeUsersHalfYear();
            $activeMonth = self::activeUsersMonthly();

-            $users = Cache::remember('api:nodeinfo:users', 43200, function() {
+            $users = Cache::remember('api:nodeinfo:users', 43200, function () {
                return User::count();
            });

-            $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-                return Status::whereLocal(true)->count();
-            });
+            $statuses = InstanceService::totalLocalStatuses();

-            $features = [ 'features' => \App\Util\Site\Config::get()['features'] ];
+            $features = ['features' => \App\Util\Site\Config::get()['features']];

            return [
                'metadata' => [
                    'nodeName' => config_cache('app.name'),
                    'software' => [
-                        'homepage'  => 'https://pixelfed.org',
-                        'repo'      => 'https://github.com/pixelfed/pixelfed',
+                        'homepage' => 'https://pixelfed.org',
+                        'repo' => 'https://github.com/pixelfed/pixelfed',
                    ],
-                    'config' => $features
+                    'config' => $features,
                ],
-                'protocols'         => [
+                'protocols' => [
                    'activitypub',
                ],
                'services' => [
-                    'inbound'  => [],
+                    'inbound' => [],
                    'outbound' => [],
                ],
                'software' => [
-                    'name'          => 'pixelfed',
-                    'version'       => config('pixelfed.version'),
+                    'name' => 'pixelfed',
+                    'version' => config('pixelfed.version'),
                ],
                'usage' => [
-                    'localPosts'    => (int) $statuses,
+                    'localPosts' => (int) $statuses,
                    'localComments' => 0,
-                    'users'         => [
-                        'total'          => (int) $users,
+                    'users' => [
+                        'total' => (int) $users,
                        'activeHalfyear' => (int) $activeHalfYear,
-                        'activeMonth'    => (int) $activeMonth,
+                        'activeMonth' => (int) $activeMonth,
                    ],
                ],
                'version' => '2.0',
            ];
        });
        $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
+
        return $res;
    }

@ -69,7 +65,7 @@ class Nodeinfo
            'links' => [
                [
                    'href' => config('pixelfed.nodeinfo.url'),
-                    'rel'  => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
+                    'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
                ],
            ],
        ];
@ -77,18 +73,18 @@ class Nodeinfo

    public static function activeUsersMonthly()
    {
-        return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
+        return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () {
            return User::withTrashed()
-                    ->select('last_active_at, updated_at')
-                    ->where('updated_at', '>', now()->subWeeks(5))
-                    ->orWhere('last_active_at', '>', now()->subWeeks(5))
-                    ->count();
+                ->select('last_active_at, updated_at')
+                ->where('updated_at', '>', now()->subWeeks(5))
+                ->orWhere('last_active_at', '>', now()->subWeeks(5))
+                ->count();
        });
    }

    public static function activeUsersHalfYear()
    {
-        return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
+        return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () {
            return User::withTrashed()
                ->select('last_active_at, updated_at')
                ->where('last_active_at', '>', now()->subMonths(6))
--- a/database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php
+++ b/database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php
@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+use App\Services\ConfigCacheService;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+        Cache::forget('api:nodeinfo');
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        //
+    }
+};