UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-25 15:55:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances

Browse source
This commit is contained in:
Daniel Supernault 2024-06-19 03:02:00 -06:00
parent bc84259a63
commit 4f2b8ed20a
No known key found for this signature in database
GPG key ID: 23740873EE6F76A1
9 changed files with 186 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
app/Console/Commands/InstanceUpdateTotalLocalPosts.php Normal file
View file

@ -0,0 +1,79 @@
<?php
namespace App\Console\Commands;
use App\Services\ConfigCacheService;
use Cache;
use DB;
use Illuminate\Console\Command;
use Storage;
class InstanceUpdateTotalLocalPosts extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:instance-update-total-local-posts';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Update the total number of local statuses/post count';
/**
* Execute the console command.
*/
public function handle()
{
$cached = $this->checkForCache();
if (! $cached) {
$this->initCache();
return;
}
$cache = $this->getCached();
if (! $cache || ! isset($cache['count'])) {
$this->error('Problem fetching cache');
return;
}
$this->updateAndCache();
Cache::forget('api:nodeinfo');
}
protected function checkForCache()
{
return Storage::exists('total_local_posts.json');
}
protected function initCache()
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count,
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
}
protected function getCached()
{
return Storage::json('total_local_posts.json');
}
protected function updateAndCache()
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count,
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
}
}

1
app/Console/Kernel.php
View file

@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
$schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
$schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
}
/**

12
app/Http/Controllers/PixelfedDirectoryController.php
View file

@ -4,13 +4,13 @@ namespace App\Http\Controllers;
use App\Models\ConfigCache;
use App\Services\AccountService;
use App\Services\InstanceService;
use App\Services\StatusService;
use App\User;
use Cache;
use Illuminate\Http\Request;
use Illuminate\Support\Str;
use Cache;
use Storage;
use App\Status;
use App\User;
class PixelfedDirectoryController extends Controller
{
@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller
'stories' => (bool) config_cache('instance.stories.enabled'),
];
$statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
return Status::whereLocal(true)->count();
});
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function() {
$statusesCount = InstanceService::totalLocalStatuses();
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
return User::count();
});
$res['stats'] = [

13
app/Services/ConfigCacheService.php
View file

@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
class ConfigCacheService
{
const CACHE_KEY = 'config_cache:_v0-key:';
const PROTECTED_KEYS = [
'filesystems.disks.s3.key',
'filesystems.disks.s3.secret',
@ -133,6 +134,8 @@ class ConfigCacheService
'filesystems.disks.spaces.url',
'filesystems.disks.spaces.endpoint',
'filesystems.disks.spaces.use_path_style_endpoint',
'instance.stats.total_local_posts',
// 'system.user_mode'
];
@ -146,7 +149,7 @@ class ConfigCacheService
$protect = false;
$protected = null;
if(in_array($key, self::PROTECTED_KEYS)) {
if (in_array($key, self::PROTECTED_KEYS)) {
$protect = true;
}
@ -154,7 +157,7 @@ class ConfigCacheService
$c = ConfigCacheModel::where('k', $key)->first();
if ($c) {
if($protect) {
if ($protect) {
return decrypt($c->v) ?? config($key);
} else {
return $c->v ?? config($key);
@ -165,7 +168,7 @@ class ConfigCacheService
return;
}
if($protect && $v) {
if ($protect && $v) {
$protected = encrypt($v);
}
@ -176,7 +179,7 @@ class ConfigCacheService
return $v;
});
} catch (Exception | QueryException $e) {
} catch (Exception|QueryException $e) {
return config($key);
}
}
@ -187,7 +190,7 @@ class ConfigCacheService
$protect = false;
$protected = null;
if(in_array($key, self::PROTECTED_KEYS)) {
if (in_array($key, self::PROTECTED_KEYS)) {
$protect = true;
$protected = encrypt($val);
}

7
app/Services/InstanceService.php
View file

@ -18,6 +18,8 @@ class InstanceService
const CACHE_KEY_STATS = 'pf:services:instances:stats';
const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
@ -96,6 +98,11 @@ class InstanceService
return true;
}
public static function totalLocalStatuses()
{
return config_cache('instance.stats.total_local_posts');
}
public static function headerBlurhash()
{
return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {

5
app/Services/LandingService.php
View file

@ -2,7 +2,6 @@
namespace App\Services;
use App\Status;
use App\User;
use App\Util\Site\Nodeinfo;
use Illuminate\Support\Facades\Cache;
@ -18,9 +17,7 @@ class LandingService
return User::count();
});
$postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
return Status::whereLocal(true)->count();
});
$postCount = InstanceService::totalLocalStatuses();
$contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
if (config_cache('instance.admin.pid')) {

54
app/Services/StatusService.php
View file

@ -2,15 +2,11 @@
namespace App\Services;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use DB;
use App\Status;
use App\Transformer\Api\StatusStatelessTransformer;
use App\Transformer\Api\StatusTransformer;
use Illuminate\Support\Facades\Cache;
use League\Fractal;
use League\Fractal\Serializer\ArraySerializer;
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
class StatusService
{
@ -19,18 +15,19 @@ class StatusService
public static function key($id, $publicOnly = true)
{
$p = $publicOnly ? 'pub:' : 'all:';
return self::CACHE_KEY . $p . $id;
return self::CACHE_KEY.$p.$id;
}
public static function get($id, $publicOnly = true, $mastodonMode = false)
{
$res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
if($publicOnly) {
$res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) {
if ($publicOnly) {
$status = Status::whereScope('public')->find($id);
} else {
$status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id);
}
if(!$status) {
if (! $status) {
return null;
}
$fractal = new Fractal\Manager();
@ -38,32 +35,34 @@ class StatusService
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
$res = $fractal->createData($resource)->toArray();
$res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
if(isset($res['_pid'])) {
if (isset($res['_pid'])) {
unset($res['account']);
}
return $res;
});
if($res && isset($res['_pid'])) {
if ($res && isset($res['_pid'])) {
$res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
unset($res['_pid']);
}
return $res;
}
public static function getMastodon($id, $publicOnly = true)
{
$status = self::get($id, $publicOnly, true);
if(!$status) {
if (! $status) {
return null;
}
if(!isset($status['account'])) {
if (! isset($status['account'])) {
return null;
}
$status['replies_count'] = $status['reply_count'];
if(config('exp.emc') == false) {
if (config('exp.emc') == false) {
return $status;
}
@ -113,28 +112,29 @@ class StatusService
{
$status = self::get($id, false);
if(!$status) {
if (! $status) {
return [
'liked' => false,
'shared' => false,
'bookmarked' => false
'bookmarked' => false,
];
}
return [
'liked' => LikeService::liked($pid, $id),
'shared' => self::isShared($id, $pid),
'bookmarked' => self::isBookmarked($id, $pid)
'bookmarked' => self::isBookmarked($id, $pid),
];
}
public static function getFull($id, $pid, $publicOnly = true)
{
$res = self::get($id, $publicOnly);
if(!$res || !isset($res['account']) || !isset($res['account']['id'])) {
if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) {
return $res;
}
$res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
return $res;
}
@ -142,31 +142,33 @@ class StatusService
{
$status = Status::whereScope('direct')->find($id);
if(!$status) {
if (! $status) {
return null;
}
$fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
return $fractal->createData($resource)->toArray();
}
public static function del($id, $purge = false)
{
if($purge) {
if ($purge) {
$status = self::get($id);
if($status && isset($status['account']) && isset($status['account']['id'])) {
Cache::forget('profile:embed:' . $status['account']['id']);
if ($status && isset($status['account']) && isset($status['account']['id'])) {
Cache::forget('profile:embed:'.$status['account']['id']);
}
Cache::forget('status:transformer:media:attachments:' . $id);
Cache::forget('status:transformer:media:attachments:'.$id);
MediaService::del($id);
Cache::forget('pf:services:sh:id:' . $id);
Cache::forget('pf:services:sh:id:'.$id);
PublicTimelineService::rem($id);
NetworkTimelineService::rem($id);
}
Cache::forget(self::key($id, false));
return Cache::forget(self::key($id));
}
@ -194,8 +196,6 @@ class StatusService
public static function totalLocalStatuses()
{
return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
return Status::whereNull('url')->count();
});
return InstanceService::totalLocalStatuses();
}
}

22
app/Util/Site/Nodeinfo.php
View file

@ -2,12 +2,9 @@
namespace App\Util\Site;
use Illuminate\Support\Facades\Cache;
use App\Like;
use App\Profile;
use App\Status;
use App\Services\InstanceService;
use App\User;
use Illuminate\Support\Str;
use Illuminate\Support\Facades\Cache;
class Nodeinfo
{
@ -17,15 +14,13 @@ class Nodeinfo
$activeHalfYear = self::activeUsersHalfYear();
$activeMonth = self::activeUsersMonthly();
$users = Cache::remember('api:nodeinfo:users', 43200, function() {
$users = Cache::remember('api:nodeinfo:users', 43200, function () {
return User::count();
});
$statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
return Status::whereLocal(true)->count();
});
$statuses = InstanceService::totalLocalStatuses();
$features = [ 'features' => \App\Util\Site\Config::get()['features'] ];
$features = ['features' => \App\Util\Site\Config::get()['features']];
return [
'metadata' => [
@ -34,7 +29,7 @@ class Nodeinfo
'homepage' => 'https://pixelfed.org',
'repo' => 'https://github.com/pixelfed/pixelfed',
],
'config' => $features
'config' => $features,
],
'protocols' => [
'activitypub',
@ -60,6 +55,7 @@ class Nodeinfo
];
});
$res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
return $res;
}
@ -77,7 +73,7 @@ class Nodeinfo
public static function activeUsersMonthly()
{
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () {
return User::withTrashed()
->select('last_active_at, updated_at')
->where('updated_at', '>', now()->subWeeks(5))
@ -88,7 +84,7 @@ class Nodeinfo
public static function activeUsersHalfYear()
{
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () {
return User::withTrashed()
->select('last_active_at, updated_at')
->where('last_active_at', '>', now()->subMonths(6))

34
database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php Normal file
View file

@ -0,0 +1,34 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Storage;
use App\Services\ConfigCacheService;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
Cache::forget('api:nodeinfo');
}
/**
* Reverse the migrations.
*/
public function down(): void
{
//
}
};