UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-18 04:21:27 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #5184 from pixelfed/staging

Browse source 
Staging
This commit is contained in:
daniel 2024-06-19 03:10:29 -06:00 committed by GitHub
commit d13e9e8e5d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 189 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -28,6 +28,9 @@
- Update StatusController, cache AP object ([a75b89b2](https://github.com/pixelfed/pixelfed/commit/a75b89b2))
- Update status embed, add support for album carousels ([f4898db9](https://github.com/pixelfed/pixelfed/commit/f4898db9))
- Update profile embeds, add support for albums ([4fd156c4](https://github.com/pixelfed/pixelfed/commit/4fd156c4))
- Update DirectMessageController, add timestamps to threads ([b24d2554](https://github.com/pixelfed/pixelfed/commit/b24d2554))
- Update DirectMessageController, add carousel entity to threads ([96f24f33](https://github.com/pixelfed/pixelfed/commit/96f24f33))
- Update and refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances ([4f2b8ed2](https://github.com/pixelfed/pixelfed/commit/4f2b8ed2))
- ([](https://github.com/pixelfed/pixelfed/commit/))
- ([](https://github.com/pixelfed/pixelfed/commit/))

79
app/Console/Commands/InstanceUpdateTotalLocalPosts.php Normal file
View file

@ -0,0 +1,79 @@
<?php
namespace App\Console\Commands;
use App\Services\ConfigCacheService;
use Cache;
use DB;
use Illuminate\Console\Command;
use Storage;
class InstanceUpdateTotalLocalPosts extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:instance-update-total-local-posts';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Update the total number of local statuses/post count';
/**
* Execute the console command.
*/
public function handle()
{
$cached = $this->checkForCache();
if (! $cached) {
$this->initCache();
return;
}
$cache = $this->getCached();
if (! $cache || ! isset($cache['count'])) {
$this->error('Problem fetching cache');
return;
}
$this->updateAndCache();
Cache::forget('api:nodeinfo');
}
protected function checkForCache()
{
return Storage::exists('total_local_posts.json');
}
protected function initCache()
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count,
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
}
protected function getCached()
{
return Storage::json('total_local_posts.json');
}
protected function updateAndCache()
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count,
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
}
}

1
app/Console/Kernel.php
View file

@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
$schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
$schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
}
/**

12
app/Http/Controllers/PixelfedDirectoryController.php
View file

@ -4,13 +4,13 @@ namespace App\Http\Controllers;
use App\Models\ConfigCache;
use App\Services\AccountService;
use App\Services\InstanceService;
use App\Services\StatusService;
use App\User;
use Cache;
use Illuminate\Http\Request;
use Illuminate\Support\Str;
use Cache;
use Storage;
use App\Status;
use App\User;
class PixelfedDirectoryController extends Controller
{
@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller
'stories' => (bool) config_cache('instance.stories.enabled'),
];
$statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
return Status::whereLocal(true)->count();
});
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function() {
$statusesCount = InstanceService::totalLocalStatuses();
$usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
return User::count();
});
$res['stats'] = [

13
app/Services/ConfigCacheService.php
View file

@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
class ConfigCacheService
{
const CACHE_KEY = 'config_cache:_v0-key:';
const PROTECTED_KEYS = [
'filesystems.disks.s3.key',
'filesystems.disks.s3.secret',
@ -133,6 +134,8 @@ class ConfigCacheService
'filesystems.disks.spaces.url',
'filesystems.disks.spaces.endpoint',
'filesystems.disks.spaces.use_path_style_endpoint',
'instance.stats.total_local_posts',
// 'system.user_mode'
];
@ -146,7 +149,7 @@ class ConfigCacheService
$protect = false;
$protected = null;
if(in_array($key, self::PROTECTED_KEYS)) {
if (in_array($key, self::PROTECTED_KEYS)) {
$protect = true;
}
@ -154,7 +157,7 @@ class ConfigCacheService
$c = ConfigCacheModel::where('k', $key)->first();
if ($c) {
if($protect) {
if ($protect) {
return decrypt($c->v) ?? config($key);
} else {
return $c->v ?? config($key);
@ -165,7 +168,7 @@ class ConfigCacheService
return;
}
if($protect && $v) {
if ($protect && $v) {
$protected = encrypt($v);
}
@ -176,7 +179,7 @@ class ConfigCacheService
return $v;
});
} catch (Exception | QueryException $e) {
} catch (Exception|QueryException $e) {
return config($key);
}
}
@ -187,7 +190,7 @@ class ConfigCacheService
$protect = false;
$protected = null;
if(in_array($key, self::PROTECTED_KEYS)) {
if (in_array($key, self::PROTECTED_KEYS)) {
$protect = true;
$protected = encrypt($val);
}

7
app/Services/InstanceService.php
View file

@ -18,6 +18,8 @@ class InstanceService
const CACHE_KEY_STATS = 'pf:services:instances:stats';
const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
@ -96,6 +98,11 @@ class InstanceService
return true;
}
public static function totalLocalStatuses()
{
return config_cache('instance.stats.total_local_posts');
}
public static function headerBlurhash()
{
return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {

5
app/Services/LandingService.php
View file

@ -2,7 +2,6 @@
namespace App\Services;
use App\Status;
use App\User;
use App\Util\Site\Nodeinfo;
use Illuminate\Support\Facades\Cache;
@ -18,9 +17,7 @@ class LandingService
return User::count();
});
$postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
return Status::whereLocal(true)->count();
});
$postCount = InstanceService::totalLocalStatuses();
$contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
if (config_cache('instance.admin.pid')) {

54
app/Services/StatusService.php
View file

@ -2,15 +2,11 @@
namespace App\Services;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use DB;
use App\Status;
use App\Transformer\Api\StatusStatelessTransformer;
use App\Transformer\Api\StatusTransformer;
use Illuminate\Support\Facades\Cache;
use League\Fractal;
use League\Fractal\Serializer\ArraySerializer;
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
class StatusService
{
@ -19,18 +15,19 @@ class StatusService
public static function key($id, $publicOnly = true)
{
$p = $publicOnly ? 'pub:' : 'all:';
return self::CACHE_KEY . $p . $id;
return self::CACHE_KEY.$p.$id;
}
public static function get($id, $publicOnly = true, $mastodonMode = false)
{
$res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
if($publicOnly) {
$res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) {
if ($publicOnly) {
$status = Status::whereScope('public')->find($id);
} else {
$status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id);
}
if(!$status) {
if (! $status) {
return null;
}
$fractal = new Fractal\Manager();
@ -38,32 +35,34 @@ class StatusService
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
$res = $fractal->createData($resource)->toArray();
$res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
if(isset($res['_pid'])) {
if (isset($res['_pid'])) {
unset($res['account']);
}
return $res;
});
if($res && isset($res['_pid'])) {
if ($res && isset($res['_pid'])) {
$res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
unset($res['_pid']);
}
return $res;
}
public static function getMastodon($id, $publicOnly = true)
{
$status = self::get($id, $publicOnly, true);
if(!$status) {
if (! $status) {
return null;
}
if(!isset($status['account'])) {
if (! isset($status['account'])) {
return null;
}
$status['replies_count'] = $status['reply_count'];
if(config('exp.emc') == false) {
if (config('exp.emc') == false) {
return $status;
}
@ -113,28 +112,29 @@ class StatusService
{
$status = self::get($id, false);
if(!$status) {
if (! $status) {
return [
'liked' => false,
'shared' => false,
'bookmarked' => false
'bookmarked' => false,
];
}
return [
'liked' => LikeService::liked($pid, $id),
'shared' => self::isShared($id, $pid),
'bookmarked' => self::isBookmarked($id, $pid)
'bookmarked' => self::isBookmarked($id, $pid),
];
}
public static function getFull($id, $pid, $publicOnly = true)
{
$res = self::get($id, $publicOnly);
if(!$res || !isset($res['account']) || !isset($res['account']['id'])) {
if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) {
return $res;
}
$res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
return $res;
}
@ -142,31 +142,33 @@ class StatusService
{
$status = Status::whereScope('direct')->find($id);
if(!$status) {
if (! $status) {
return null;
}
$fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
return $fractal->createData($resource)->toArray();
}
public static function del($id, $purge = false)
{
if($purge) {
if ($purge) {
$status = self::get($id);
if($status && isset($status['account']) && isset($status['account']['id'])) {
Cache::forget('profile:embed:' . $status['account']['id']);
if ($status && isset($status['account']) && isset($status['account']['id'])) {
Cache::forget('profile:embed:'.$status['account']['id']);
}
Cache::forget('status:transformer:media:attachments:' . $id);
Cache::forget('status:transformer:media:attachments:'.$id);
MediaService::del($id);
Cache::forget('pf:services:sh:id:' . $id);
Cache::forget('pf:services:sh:id:'.$id);
PublicTimelineService::rem($id);
NetworkTimelineService::rem($id);
}
Cache::forget(self::key($id, false));
return Cache::forget(self::key($id));
}
@ -194,8 +196,6 @@ class StatusService
public static function totalLocalStatuses()
{
return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
return Status::whereNull('url')->count();
});
return InstanceService::totalLocalStatuses();
}
}

52
app/Util/Site/Nodeinfo.php
View file

@ -2,12 +2,9 @@
namespace App\Util\Site;
use Illuminate\Support\Facades\Cache;
use App\Like;
use App\Profile;
use App\Status;
use App\Services\InstanceService;
use App\User;
use Illuminate\Support\Str;
use Illuminate\Support\Facades\Cache;
class Nodeinfo
{
@ -17,49 +14,48 @@ class Nodeinfo
$activeHalfYear = self::activeUsersHalfYear();
$activeMonth = self::activeUsersMonthly();
$users = Cache::remember('api:nodeinfo:users', 43200, function() {
$users = Cache::remember('api:nodeinfo:users', 43200, function () {
return User::count();
});
$statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
return Status::whereLocal(true)->count();
});
$statuses = InstanceService::totalLocalStatuses();
$features = [ 'features' => \App\Util\Site\Config::get()['features'] ];
$features = ['features' => \App\Util\Site\Config::get()['features']];
return [
'metadata' => [
'nodeName' => config_cache('app.name'),
'software' => [
'homepage' => 'https://pixelfed.org',
'repo' => 'https://github.com/pixelfed/pixelfed',
'homepage' => 'https://pixelfed.org',
'repo' => 'https://github.com/pixelfed/pixelfed',
],
'config' => $features
'config' => $features,
],
'protocols' => [
'protocols' => [
'activitypub',
],
'services' => [
'inbound' => [],
'inbound' => [],
'outbound' => [],
],
'software' => [
'name' => 'pixelfed',
'version' => config('pixelfed.version'),
'name' => 'pixelfed',
'version' => config('pixelfed.version'),
],
'usage' => [
'localPosts' => (int) $statuses,
'localPosts' => (int) $statuses,
'localComments' => 0,
'users' => [
'total' => (int) $users,
'users' => [
'total' => (int) $users,
'activeHalfyear' => (int) $activeHalfYear,
'activeMonth' => (int) $activeMonth,
'activeMonth' => (int) $activeMonth,
],
],
'version' => '2.0',
];
});
$res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
return $res;
}
@ -69,7 +65,7 @@ class Nodeinfo
'links' => [
[
'href' => config('pixelfed.nodeinfo.url'),
'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
],
],
];
@ -77,18 +73,18 @@ class Nodeinfo
public static function activeUsersMonthly()
{
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () {
return User::withTrashed()
->select('last_active_at, updated_at')
->where('updated_at', '>', now()->subWeeks(5))
->orWhere('last_active_at', '>', now()->subWeeks(5))
->count();
->select('last_active_at, updated_at')
->where('updated_at', '>', now()->subWeeks(5))
->orWhere('last_active_at', '>', now()->subWeeks(5))
->count();
});
}
public static function activeUsersHalfYear()
{
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () {
return User::withTrashed()
->select('last_active_at, updated_at')
->where('last_active_at', '>', now()->subMonths(6))

34
database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php Normal file
View file

@ -0,0 +1,34 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Storage;
use App\Services\ConfigCacheService;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
$count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
$res = [
'count' => $count
];
Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
Cache::forget('api:nodeinfo');
}
/**
* Reverse the migrations.
*/
public function down(): void
{
//
}
};