Add NetworkTimelineService cache

app/Http/Controllers/PublicApiController.php

@@ -32,6 +32,7 @@ use App\Services\{
     LikeService,
     PublicTimelineService,
     ProfileService,
+    NetworkTimelineService,
     ReblogService,
     RelationshipService,
     StatusService,
@@ -608,6 +609,7 @@ class PublicApiController extends Controller
 
         $filtered = $user ? UserFilterService::filters($user->profile_id) : [];
 
+        if(config('instance.timeline.network.cached') == false) {
 	        if($min || $max) {
 	            $dir = $min ? '>' : '<';
 	            $id = $min ?? $max;
@@ -662,6 +664,38 @@ class PublicApiController extends Controller
 	                          });
 	                $res = $timeline->toArray();
 	        }
+	    } else {
+            Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
+                if(NetworkTimelineService::count() == 0) {
+                    NetworkTimelineService::warmCache(true, 400);
+                }
+            });
+
+            if ($max) {
+                $feed = NetworkTimelineService::getRankedMaxId($max, $limit);
+            } else if ($min) {
+                $feed = NetworkTimelineService::getRankedMinId($min, $limit);
+            } else {
+                $feed = NetworkTimelineService::get(0, $limit);
+            }
+
+            $res = collect($feed)
+            ->map(function($k) use($user) {
+                $status = StatusService::get($k);
+                if($status && isset($status['account']) && $user) {
+                    $status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
+                    $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
+                    $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
+                    $status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
+                }
+                return $status;
+            })
+            ->filter(function($s) use($filtered) {
+                return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
+            })
+            ->values()
+            ->toArray();
+        }
 
         return response()->json($res);
     }

app/Services/NetworkTimelineService.php

@@ -0,0 +1,95 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Redis;
+use App\{
+	Profile,
+	Status,
+	UserFilter
+};
+
+class NetworkTimelineService
+{
+	const CACHE_KEY = 'pf:services:timeline:network';
+
+	public static function get($start = 0, $stop = 10)
+	{
+		if($stop > 100) {
+			$stop = 100;
+		}
+
+		return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
+	}
+
+	public static function getRankedMaxId($start = null, $limit = 10)
+	{
+		if(!$start) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
+			'withscores' => true,
+			'limit' => [1, $limit]
+		]));
+	}
+
+	public static function getRankedMinId($end = null, $limit = 10)
+	{
+		if(!$end) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
+			'withscores' => true,
+			'limit' => [0, $limit]
+		]));
+	}
+
+	public static function add($val)
+	{
+		if(self::count() > config('instance.timeline.network.cache_dropoff')) {
+			if(config('database.redis.client') === 'phpredis') {
+				Redis::zpopmin(self::CACHE_KEY);
+			}
+		}
+
+		return Redis::zadd(self::CACHE_KEY, $val, $val);
+	}
+
+	public static function rem($val)
+	{
+		return Redis::zrem(self::CACHE_KEY, $val);
+	}
+
+	public static function del($val)
+	{
+		return self::rem($val);
+	}
+
+	public static function count()
+	{
+		return Redis::zcard(self::CACHE_KEY);
+	}
+
+	public static function warmCache($force = false, $limit = 100)
+	{
+		if(self::count() == 0 || $force == true) {
+			Redis::del(self::CACHE_KEY);
+			$ids = Status::whereNotNull('uri')
+				->whereScope('public')
+				->whereNull('in_reply_to_id')
+				->whereNull('reblog_of_id')
+				->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+				->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
+				->orderByDesc('created_at')
+				->limit($limit)
+				->pluck('id');
+			foreach($ids as $id) {
+				self::add($id);
+			}
+			return 1;
+		}
+		return 0;
+	}
+}

app/Util/ActivityPub/Helpers.php

@@ -32,6 +32,7 @@ use App\Services\CustomEmojiService;
 use App\Services\InstanceService;
 use App\Services\MediaPathService;
 use App\Services\MediaStorageService;
+use App\Services\NetworkTimelineService;
 use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
 use App\Util\Media\License;
@@ -490,6 +491,16 @@ class Helpers {
 			if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) {
 				StatusTagsPipeline::dispatch($activity, $status);
 			}
+
+			if( config('instance.timeline.network.cached') &&
+				$status->in_reply_to_id === null &&
+				$status->reblog_of_id === null &&
+				in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) &&
+				$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))
+			) {
+				NetworkTimelineService::add($status->id);
+			}
+
 			return $status;
 		});
 	}

config/instance.php

@@ -24,6 +24,12 @@ return [
 	'timeline' => [
 		'local' => [
 			'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
+		],
+
+		'network' => [
+			'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
+			'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
+			'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
 		]
 	],