mirror of
https://github.com/pixelfed/pixelfed.git
synced 2024-11-22 06:21:27 +00:00
Refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances
This commit is contained in:
parent
bc84259a63
commit
4f2b8ed20a
9 changed files with 186 additions and 71 deletions
79
app/Console/Commands/InstanceUpdateTotalLocalPosts.php
Normal file
79
app/Console/Commands/InstanceUpdateTotalLocalPosts.php
Normal file
|
@ -0,0 +1,79 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\ConfigCacheService;
|
||||
use Cache;
|
||||
use DB;
|
||||
use Illuminate\Console\Command;
|
||||
use Storage;
|
||||
|
||||
class InstanceUpdateTotalLocalPosts extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:instance-update-total-local-posts';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Update the total number of local statuses/post count';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$cached = $this->checkForCache();
|
||||
if (! $cached) {
|
||||
$this->initCache();
|
||||
|
||||
return;
|
||||
}
|
||||
$cache = $this->getCached();
|
||||
if (! $cache || ! isset($cache['count'])) {
|
||||
$this->error('Problem fetching cache');
|
||||
|
||||
return;
|
||||
}
|
||||
$this->updateAndCache();
|
||||
Cache::forget('api:nodeinfo');
|
||||
|
||||
}
|
||||
|
||||
protected function checkForCache()
|
||||
{
|
||||
return Storage::exists('total_local_posts.json');
|
||||
}
|
||||
|
||||
protected function initCache()
|
||||
{
|
||||
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
|
||||
$res = [
|
||||
'count' => $count,
|
||||
];
|
||||
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
|
||||
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
|
||||
}
|
||||
|
||||
protected function getCached()
|
||||
{
|
||||
return Storage::json('total_local_posts.json');
|
||||
}
|
||||
|
||||
protected function updateAndCache()
|
||||
{
|
||||
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
|
||||
$res = [
|
||||
'count' => $count,
|
||||
];
|
||||
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
|
||||
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
|
||||
|
||||
}
|
||||
}
|
|
@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
|
|||
$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
|
||||
$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
|
||||
$schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
|
||||
$schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -4,13 +4,13 @@ namespace App\Http\Controllers;
|
|||
|
||||
use App\Models\ConfigCache;
|
||||
use App\Services\AccountService;
|
||||
use App\Services\InstanceService;
|
||||
use App\Services\StatusService;
|
||||
use App\User;
|
||||
use Cache;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Str;
|
||||
use Cache;
|
||||
use Storage;
|
||||
use App\Status;
|
||||
use App\User;
|
||||
|
||||
class PixelfedDirectoryController extends Controller
|
||||
{
|
||||
|
@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller
|
|||
'stories' => (bool) config_cache('instance.stories.enabled'),
|
||||
];
|
||||
|
||||
$statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
|
||||
return Status::whereLocal(true)->count();
|
||||
});
|
||||
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function() {
|
||||
$statusesCount = InstanceService::totalLocalStatuses();
|
||||
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
|
||||
return User::count();
|
||||
});
|
||||
$res['stats'] = [
|
||||
|
|
|
@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
|
|||
class ConfigCacheService
|
||||
{
|
||||
const CACHE_KEY = 'config_cache:_v0-key:';
|
||||
|
||||
const PROTECTED_KEYS = [
|
||||
'filesystems.disks.s3.key',
|
||||
'filesystems.disks.s3.secret',
|
||||
|
@ -133,6 +134,8 @@ class ConfigCacheService
|
|||
'filesystems.disks.spaces.url',
|
||||
'filesystems.disks.spaces.endpoint',
|
||||
'filesystems.disks.spaces.use_path_style_endpoint',
|
||||
|
||||
'instance.stats.total_local_posts',
|
||||
// 'system.user_mode'
|
||||
];
|
||||
|
||||
|
@ -146,7 +149,7 @@ class ConfigCacheService
|
|||
|
||||
$protect = false;
|
||||
$protected = null;
|
||||
if(in_array($key, self::PROTECTED_KEYS)) {
|
||||
if (in_array($key, self::PROTECTED_KEYS)) {
|
||||
$protect = true;
|
||||
}
|
||||
|
||||
|
@ -154,7 +157,7 @@ class ConfigCacheService
|
|||
$c = ConfigCacheModel::where('k', $key)->first();
|
||||
|
||||
if ($c) {
|
||||
if($protect) {
|
||||
if ($protect) {
|
||||
return decrypt($c->v) ?? config($key);
|
||||
} else {
|
||||
return $c->v ?? config($key);
|
||||
|
@ -165,7 +168,7 @@ class ConfigCacheService
|
|||
return;
|
||||
}
|
||||
|
||||
if($protect && $v) {
|
||||
if ($protect && $v) {
|
||||
$protected = encrypt($v);
|
||||
}
|
||||
|
||||
|
@ -176,7 +179,7 @@ class ConfigCacheService
|
|||
|
||||
return $v;
|
||||
});
|
||||
} catch (Exception | QueryException $e) {
|
||||
} catch (Exception|QueryException $e) {
|
||||
return config($key);
|
||||
}
|
||||
}
|
||||
|
@ -187,7 +190,7 @@ class ConfigCacheService
|
|||
|
||||
$protect = false;
|
||||
$protected = null;
|
||||
if(in_array($key, self::PROTECTED_KEYS)) {
|
||||
if (in_array($key, self::PROTECTED_KEYS)) {
|
||||
$protect = true;
|
||||
$protected = encrypt($val);
|
||||
}
|
||||
|
|
|
@ -18,6 +18,8 @@ class InstanceService
|
|||
|
||||
const CACHE_KEY_STATS = 'pf:services:instances:stats';
|
||||
|
||||
const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
|
||||
|
||||
const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
|
||||
|
||||
const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
|
||||
|
@ -96,6 +98,11 @@ class InstanceService
|
|||
return true;
|
||||
}
|
||||
|
||||
public static function totalLocalStatuses()
|
||||
{
|
||||
return config_cache('instance.stats.total_local_posts');
|
||||
}
|
||||
|
||||
public static function headerBlurhash()
|
||||
{
|
||||
return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {
|
||||
|
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Status;
|
||||
use App\User;
|
||||
use App\Util\Site\Nodeinfo;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
@ -18,9 +17,7 @@ class LandingService
|
|||
return User::count();
|
||||
});
|
||||
|
||||
$postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
|
||||
return Status::whereLocal(true)->count();
|
||||
});
|
||||
$postCount = InstanceService::totalLocalStatuses();
|
||||
|
||||
$contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
|
||||
if (config_cache('instance.admin.pid')) {
|
||||
|
|
|
@ -2,15 +2,11 @@
|
|||
|
||||
namespace App\Services;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Redis;
|
||||
use DB;
|
||||
use App\Status;
|
||||
use App\Transformer\Api\StatusStatelessTransformer;
|
||||
use App\Transformer\Api\StatusTransformer;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use League\Fractal;
|
||||
use League\Fractal\Serializer\ArraySerializer;
|
||||
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
|
||||
|
||||
class StatusService
|
||||
{
|
||||
|
@ -19,18 +15,19 @@ class StatusService
|
|||
public static function key($id, $publicOnly = true)
|
||||
{
|
||||
$p = $publicOnly ? 'pub:' : 'all:';
|
||||
return self::CACHE_KEY . $p . $id;
|
||||
|
||||
return self::CACHE_KEY.$p.$id;
|
||||
}
|
||||
|
||||
public static function get($id, $publicOnly = true, $mastodonMode = false)
|
||||
{
|
||||
$res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
|
||||
if($publicOnly) {
|
||||
$res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) {
|
||||
if ($publicOnly) {
|
||||
$status = Status::whereScope('public')->find($id);
|
||||
} else {
|
||||
$status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id);
|
||||
}
|
||||
if(!$status) {
|
||||
if (! $status) {
|
||||
return null;
|
||||
}
|
||||
$fractal = new Fractal\Manager();
|
||||
|
@ -38,32 +35,34 @@ class StatusService
|
|||
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
|
||||
$res = $fractal->createData($resource)->toArray();
|
||||
$res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
|
||||
if(isset($res['_pid'])) {
|
||||
if (isset($res['_pid'])) {
|
||||
unset($res['account']);
|
||||
}
|
||||
|
||||
return $res;
|
||||
});
|
||||
if($res && isset($res['_pid'])) {
|
||||
if ($res && isset($res['_pid'])) {
|
||||
$res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
|
||||
unset($res['_pid']);
|
||||
}
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
public static function getMastodon($id, $publicOnly = true)
|
||||
{
|
||||
$status = self::get($id, $publicOnly, true);
|
||||
if(!$status) {
|
||||
if (! $status) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if(!isset($status['account'])) {
|
||||
if (! isset($status['account'])) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$status['replies_count'] = $status['reply_count'];
|
||||
|
||||
if(config('exp.emc') == false) {
|
||||
if (config('exp.emc') == false) {
|
||||
return $status;
|
||||
}
|
||||
|
||||
|
@ -113,28 +112,29 @@ class StatusService
|
|||
{
|
||||
$status = self::get($id, false);
|
||||
|
||||
if(!$status) {
|
||||
if (! $status) {
|
||||
return [
|
||||
'liked' => false,
|
||||
'shared' => false,
|
||||
'bookmarked' => false
|
||||
'bookmarked' => false,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'liked' => LikeService::liked($pid, $id),
|
||||
'shared' => self::isShared($id, $pid),
|
||||
'bookmarked' => self::isBookmarked($id, $pid)
|
||||
'bookmarked' => self::isBookmarked($id, $pid),
|
||||
];
|
||||
}
|
||||
|
||||
public static function getFull($id, $pid, $publicOnly = true)
|
||||
{
|
||||
$res = self::get($id, $publicOnly);
|
||||
if(!$res || !isset($res['account']) || !isset($res['account']['id'])) {
|
||||
if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) {
|
||||
return $res;
|
||||
}
|
||||
$res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
|
@ -142,31 +142,33 @@ class StatusService
|
|||
{
|
||||
$status = Status::whereScope('direct')->find($id);
|
||||
|
||||
if(!$status) {
|
||||
if (! $status) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$fractal = new Fractal\Manager();
|
||||
$fractal->setSerializer(new ArraySerializer());
|
||||
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
|
||||
|
||||
return $fractal->createData($resource)->toArray();
|
||||
}
|
||||
|
||||
public static function del($id, $purge = false)
|
||||
{
|
||||
if($purge) {
|
||||
if ($purge) {
|
||||
$status = self::get($id);
|
||||
if($status && isset($status['account']) && isset($status['account']['id'])) {
|
||||
Cache::forget('profile:embed:' . $status['account']['id']);
|
||||
if ($status && isset($status['account']) && isset($status['account']['id'])) {
|
||||
Cache::forget('profile:embed:'.$status['account']['id']);
|
||||
}
|
||||
Cache::forget('status:transformer:media:attachments:' . $id);
|
||||
Cache::forget('status:transformer:media:attachments:'.$id);
|
||||
MediaService::del($id);
|
||||
Cache::forget('pf:services:sh:id:' . $id);
|
||||
Cache::forget('pf:services:sh:id:'.$id);
|
||||
PublicTimelineService::rem($id);
|
||||
NetworkTimelineService::rem($id);
|
||||
}
|
||||
|
||||
Cache::forget(self::key($id, false));
|
||||
|
||||
return Cache::forget(self::key($id));
|
||||
}
|
||||
|
||||
|
@ -194,8 +196,6 @@ class StatusService
|
|||
|
||||
public static function totalLocalStatuses()
|
||||
{
|
||||
return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
|
||||
return Status::whereNull('url')->count();
|
||||
});
|
||||
return InstanceService::totalLocalStatuses();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,12 +2,9 @@
|
|||
|
||||
namespace App\Util\Site;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use App\Like;
|
||||
use App\Profile;
|
||||
use App\Status;
|
||||
use App\Services\InstanceService;
|
||||
use App\User;
|
||||
use Illuminate\Support\Str;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
||||
class Nodeinfo
|
||||
{
|
||||
|
@ -17,49 +14,48 @@ class Nodeinfo
|
|||
$activeHalfYear = self::activeUsersHalfYear();
|
||||
$activeMonth = self::activeUsersMonthly();
|
||||
|
||||
$users = Cache::remember('api:nodeinfo:users', 43200, function() {
|
||||
$users = Cache::remember('api:nodeinfo:users', 43200, function () {
|
||||
return User::count();
|
||||
});
|
||||
|
||||
$statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
|
||||
return Status::whereLocal(true)->count();
|
||||
});
|
||||
$statuses = InstanceService::totalLocalStatuses();
|
||||
|
||||
$features = [ 'features' => \App\Util\Site\Config::get()['features'] ];
|
||||
$features = ['features' => \App\Util\Site\Config::get()['features']];
|
||||
|
||||
return [
|
||||
'metadata' => [
|
||||
'nodeName' => config_cache('app.name'),
|
||||
'software' => [
|
||||
'homepage' => 'https://pixelfed.org',
|
||||
'repo' => 'https://github.com/pixelfed/pixelfed',
|
||||
'homepage' => 'https://pixelfed.org',
|
||||
'repo' => 'https://github.com/pixelfed/pixelfed',
|
||||
],
|
||||
'config' => $features
|
||||
'config' => $features,
|
||||
],
|
||||
'protocols' => [
|
||||
'protocols' => [
|
||||
'activitypub',
|
||||
],
|
||||
'services' => [
|
||||
'inbound' => [],
|
||||
'inbound' => [],
|
||||
'outbound' => [],
|
||||
],
|
||||
'software' => [
|
||||
'name' => 'pixelfed',
|
||||
'version' => config('pixelfed.version'),
|
||||
'name' => 'pixelfed',
|
||||
'version' => config('pixelfed.version'),
|
||||
],
|
||||
'usage' => [
|
||||
'localPosts' => (int) $statuses,
|
||||
'localPosts' => (int) $statuses,
|
||||
'localComments' => 0,
|
||||
'users' => [
|
||||
'total' => (int) $users,
|
||||
'users' => [
|
||||
'total' => (int) $users,
|
||||
'activeHalfyear' => (int) $activeHalfYear,
|
||||
'activeMonth' => (int) $activeMonth,
|
||||
'activeMonth' => (int) $activeMonth,
|
||||
],
|
||||
],
|
||||
'version' => '2.0',
|
||||
];
|
||||
});
|
||||
$res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
|
@ -69,7 +65,7 @@ class Nodeinfo
|
|||
'links' => [
|
||||
[
|
||||
'href' => config('pixelfed.nodeinfo.url'),
|
||||
'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
|
||||
'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
|
||||
],
|
||||
],
|
||||
];
|
||||
|
@ -77,18 +73,18 @@ class Nodeinfo
|
|||
|
||||
public static function activeUsersMonthly()
|
||||
{
|
||||
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
|
||||
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () {
|
||||
return User::withTrashed()
|
||||
->select('last_active_at, updated_at')
|
||||
->where('updated_at', '>', now()->subWeeks(5))
|
||||
->orWhere('last_active_at', '>', now()->subWeeks(5))
|
||||
->count();
|
||||
->select('last_active_at, updated_at')
|
||||
->where('updated_at', '>', now()->subWeeks(5))
|
||||
->orWhere('last_active_at', '>', now()->subWeeks(5))
|
||||
->count();
|
||||
});
|
||||
}
|
||||
|
||||
public static function activeUsersHalfYear()
|
||||
{
|
||||
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
|
||||
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () {
|
||||
return User::withTrashed()
|
||||
->select('last_active_at, updated_at')
|
||||
->where('last_active_at', '>', now()->subMonths(6))
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use App\Services\ConfigCacheService;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
|
||||
$res = [
|
||||
'count' => $count
|
||||
];
|
||||
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
|
||||
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
|
||||
Cache::forget('api:nodeinfo');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
//
|
||||
}
|
||||
};
|
Loading…
Reference in a new issue