UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-29 09:43:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add NetworkTimelineService cache

Browse source
This commit is contained in:
Daniel Supernault 2022-06-09 04:15:23 -06:00
parent 9c17def4a0
commit 1310d95cdb
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
4 changed files with 199 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
app/Http/Controllers/PublicApiController.php
View file

@ -32,6 +32,7 @@ use App\Services\{
LikeService, LikeService,
PublicTimelineService, PublicTimelineService,
ProfileService, ProfileService,
NetworkTimelineService,
ReblogService, ReblogService,
RelationshipService, RelationshipService,
StatusService, StatusService,
@ -608,59 +609,92 @@ class PublicApiController extends Controller
$filtered = $user ? UserFilterService::filters($user->profile_id) : []; $filtered = $user ? UserFilterService::filters($user->profile_id) : [];
if($min || $max) { if(config('instance.timeline.network.cached') == false) {
$dir = $min ? '>' : '<'; if($min || $max) {
$id = $min ?? $max; $dir = $min ? '>' : '<';
$timeline = Status::select( $id = $min ?? $max;
'id', $timeline = Status::select(
'uri', 'id',
'type', 'uri',
'scope', 'type',
'created_at', 'scope',
) 'created_at',
->where('id', $dir, $id) )
->whereNull(['in_reply_to_id', 'reblog_of_id']) ->where('id', $dir, $id)
->whereNotIn('profile_id', $filtered) ->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) ->whereNotIn('profile_id', $filtered)
->whereNotNull('uri') ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereScope('public') ->whereNotNull('uri')
->where('id', '>', $amin) ->whereScope('public')
->orderBy('created_at', 'desc') ->where('id', '>', $amin)
->limit($limit) ->orderBy('created_at', 'desc')
->get() ->limit($limit)
->map(function($s) use ($user) { ->get()
$status = StatusService::get($s->id); ->map(function($s) use ($user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); $status = StatusService::get($s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
return $status; $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
}); return $status;
$res = $timeline->toArray(); });
} else { $res = $timeline->toArray();
$timeline = Status::select( } else {
'id', $timeline = Status::select(
'uri', 'id',
'type', 'uri',
'scope', 'type',
'created_at', 'scope',
) 'created_at',
->whereNull(['in_reply_to_id', 'reblog_of_id']) )
->whereNotIn('profile_id', $filtered) ->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) ->whereNotIn('profile_id', $filtered)
->whereNotNull('uri') ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereScope('public') ->whereNotNull('uri')
->where('id', '>', $amin) ->whereScope('public')
->orderBy('created_at', 'desc') ->where('id', '>', $amin)
->limit($limit) ->orderBy('created_at', 'desc')
->get() ->limit($limit)
->map(function($s) use ($user) { ->get()
$status = StatusService::get($s->id); ->map(function($s) use ($user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); $status = StatusService::get($s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
return $status; $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
}); return $status;
$res = $timeline->toArray(); });
$res = $timeline->toArray();
}
} else {
Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
if(NetworkTimelineService::count() == 0) {
NetworkTimelineService::warmCache(true, 400);
}
});
if ($max) {
$feed = NetworkTimelineService::getRankedMaxId($max, $limit);
} else if ($min) {
$feed = NetworkTimelineService::getRankedMinId($min, $limit);
} else {
$feed = NetworkTimelineService::get(0, $limit);
}
$res = collect($feed)
->map(function($k) use($user) {
$status = StatusService::get($k);
if($status && isset($status['account']) && $user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
$status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
}
return $status;
})
->filter(function($s) use($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values()
->toArray();
} }
return response()->json($res); return response()->json($res);

95
app/Services/NetworkTimelineService.php Normal file
View file

@ -0,0 +1,95 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Redis;
use App\{
Profile,
Status,
UserFilter
};
class NetworkTimelineService
{
const CACHE_KEY = 'pf:services:timeline:network';
public static function get($start = 0, $stop = 10)
{
if($stop > 100) {
$stop = 100;
}
return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
}
public static function getRankedMaxId($start = null, $limit = 10)
{
if(!$start) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit]
]));
}
public static function getRankedMinId($end = null, $limit = 10)
{
if(!$end) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}
public static function add($val)
{
if(self::count() > config('instance.timeline.network.cache_dropoff')) {
if(config('database.redis.client') === 'phpredis') {
Redis::zpopmin(self::CACHE_KEY);
}
}
return Redis::zadd(self::CACHE_KEY, $val, $val);
}
public static function rem($val)
{
return Redis::zrem(self::CACHE_KEY, $val);
}
public static function del($val)
{
return self::rem($val);
}
public static function count()
{
return Redis::zcard(self::CACHE_KEY);
}
public static function warmCache($force = false, $limit = 100)
{
if(self::count() == 0 || $force == true) {
Redis::del(self::CACHE_KEY);
$ids = Status::whereNotNull('uri')
->whereScope('public')
->whereNull('in_reply_to_id')
->whereNull('reblog_of_id')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
->orderByDesc('created_at')
->limit($limit)
->pluck('id');
foreach($ids as $id) {
self::add($id);
}
return 1;
}
return 0;
}
}

11
app/Util/ActivityPub/Helpers.php
View file

@ -32,6 +32,7 @@ use App\Services\CustomEmojiService;
use App\Services\InstanceService; use App\Services\InstanceService;
use App\Services\MediaPathService; use App\Services\MediaPathService;
use App\Services\MediaStorageService; use App\Services\MediaStorageService;
use App\Services\NetworkTimelineService;
use App\Jobs\MediaPipeline\MediaStoragePipeline; use App\Jobs\MediaPipeline\MediaStoragePipeline;
use App\Jobs\AvatarPipeline\RemoteAvatarFetch; use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
use App\Util\Media\License; use App\Util\Media\License;
@ -490,6 +491,16 @@ class Helpers {
if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) { if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) {
StatusTagsPipeline::dispatch($activity, $status); StatusTagsPipeline::dispatch($activity, $status);
} }
if( config('instance.timeline.network.cached') &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) &&
$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))
) {
NetworkTimelineService::add($status->id);
}
return $status; return $status;
}); });
} }

6
config/instance.php
View file

@ -24,6 +24,12 @@ return [
'timeline' => [ 'timeline' => [
'local' => [ 'local' => [
'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false) 'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
],
'network' => [
'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
] ]
], ],