Merge pull request #4088 from pixelfed/staging

Staging

CHANGELOG.md

@@ -78,6 +78,7 @@
 - Update database config, add sticky flag https://laravel.com/docs/9.x/database#the-sticky-option ([10b65980](https://github.com/pixelfed/pixelfed/commit/10b65980))
 - Update profile audience to filter blocked instances ([e0c3dae3](https://github.com/pixelfed/pixelfed/commit/e0c3dae3))
 - Update SearchApiV2Service, improve query performance ([4d1f2811](https://github.com/pixelfed/pixelfed/commit/4d1f2811))
+- Update InstanceService, improve unlisted/banned network post filtering ([a0da6ec3](https://github.com/pixelfed/pixelfed/commit/a0da6ec3))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

app/Http/Controllers/Api/AdminApiController.php

@@ -9,6 +9,7 @@ use Auth, Cache, DB;
 use Carbon\Carbon;
 use App\{
     AccountInterstitial,
+    Instance,
     Like,
     Media,
     Profile,
@@ -19,9 +20,12 @@ use App\{
 use App\Services\AccountService;
 use App\Services\AdminStatsService;
 use App\Services\ConfigCacheService;
+use App\Services\InstanceService;
 use App\Services\ModLogService;
 use App\Services\StatusService;
+use App\Services\NetworkTimelineService;
 use App\Services\NotificationService;
+use App\Http\Resources\AdminInstance;
 use App\Http\Resources\AdminUser;
 
 class AdminApiController extends Controller
@@ -486,4 +490,94 @@ class AdminApiController extends Controller
             ]
         ]]);
     }
+
+    public function instances(Request $request)
+    {
+        abort_if(!$request->user(), 404);
+        abort_unless($request->user()->is_admin === 1, 404);
+        $this->validate($request, [
+            'q' => 'sometimes',
+            'sort' => 'sometimes|in:asc,desc',
+            'sort_by' => 'sometimes|in:id,status_count,user_count,domain',
+            'filter' => 'sometimes|in:all,unlisted,auto_cw,banned',
+        ]);
+
+        $q = $request->input('q');
+        $sort = $request->input('sort', 'desc') === 'asc' ? 'asc' : 'desc';
+        $sortBy = $request->input('sort_by', 'id');
+        $filter = $request->input('filter');
+
+        $res = Instance::when($q, function($query, $q) {
+                return $query->where('domain', 'like', '%' . $q . '%');
+            })
+            ->when($filter, function($query, $filter) {
+                if($filter === 'all') {
+                    return $query;
+                } else {
+                    return $query->where($filter, true);
+                }
+            })
+            ->when($sortBy, function($query, $sortBy) use($sort) {
+                return $query->orderBy($sortBy, $sort);
+            }, function($query) {
+                return $query->orderBy('id', 'desc');
+            })
+            ->cursorPaginate(10)
+            ->withQueryString();
+
+        return AdminInstance::collection($res);
+    }
+
+    public function getInstance(Request $request)
+    {
+        abort_if(!$request->user(), 404);
+        abort_unless($request->user()->is_admin === 1, 404);
+
+        $id = $request->input('id');
+        $res = Instance::findOrFail($id);
+
+        return new AdminInstance($res);
+    }
+
+    public function moderateInstance(Request $request)
+    {
+        abort_if(!$request->user(), 404);
+        abort_unless($request->user()->is_admin === 1, 404);
+
+        $this->validate($request, [
+            'id' => 'required',
+            'key' => 'required|in:unlisted,auto_cw,banned',
+            'value' => 'required'
+        ]);
+
+        $id = $request->input('id');
+        $key = $request->input('key');
+        $value = (bool) filter_var($request->input('value'), FILTER_VALIDATE_BOOLEAN);
+        $res = Instance::findOrFail($id);
+        $res->{$key} = $value;
+        $res->save();
+
+        InstanceService::refresh();
+        NetworkTimelineService::warmCache(true);
+
+        return new AdminInstance($res);
+    }
+
+    public function refreshInstanceStats(Request $request)
+    {
+        abort_if(!$request->user(), 404);
+        abort_unless($request->user()->is_admin === 1, 404);
+
+        $this->validate($request, [
+            'id' => 'required',
+        ]);
+
+        $id = $request->input('id');
+        $instance = Instance::findOrFail($id);
+        $instance->user_count = Profile::whereDomain($instance->domain)->count();
+        $instance->status_count = Profile::whereDomain($instance->domain)->leftJoin('statuses', 'profiles.id', '=', 'statuses.profile_id')->count();
+        $instance->save();
+
+        return new AdminInstance($instance);
+    }
 }

app/Http/Resources/AdminInstance.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Resources\Json\JsonResource;
+
+class AdminInstance extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @return array|\Illuminate\Contracts\Support\Arrayable|\JsonSerializable
+     */
+    public function toArray($request)
+    {
+        return [
+            'id' => $this->id,
+            'domain' => $this->domain,
+            'software' => $this->software,
+            'unlisted' => (bool) $this->unlisted,
+            'auto_cw' => (bool) $this->auto_cw,
+            'banned' => (bool) $this->banned,
+            'user_count' => $this->user_count,
+            'status_count' => $this->status_count,
+            'last_crawled_at' => $this->last_crawled_at,
+            'actors_last_synced_at' => $this->actors_last_synced_at,
+            'created_at' => $this->created_at,
+        ];
+    }
+}

app/Services/InstanceService.php

@@ -7,13 +7,14 @@ use App\Instance;
 
 class InstanceService
 {
+    const CACHE_KEY_BY_DOMAIN = 'pf:services:instance:by_domain:';
 	const CACHE_KEY_BANNED_DOMAINS = 'instances:banned:domains';
 	const CACHE_KEY_UNLISTED_DOMAINS = 'instances:unlisted:domains';
 	const CACHE_KEY_NSFW_DOMAINS = 'instances:auto_cw:domains';
 
 	public static function getByDomain($domain)
 	{
-		return Cache::remember('pf:services:instance:by_domain:'.$domain, 3600, function() use($domain) {
+		return Cache::remember(self::CACHE_KEY_BY_DOMAIN.$domain, 3600, function() use($domain) {
 			return Instance::whereDomain($domain)->first();
 		});
 	}
@@ -50,4 +51,17 @@ class InstanceService
 			return $instance->software;
 		});
 	}
+
+    public static function refresh()
+    {
+        Cache::forget(self::CACHE_KEY_BANNED_DOMAINS);
+        Cache::forget(self::CACHE_KEY_UNLISTED_DOMAINS);
+        Cache::forget(self::CACHE_KEY_NSFW_DOMAINS);
+
+        self::getBannedDomains();
+        self::getUnlistedDomains();
+        self::getNsfwDomains();
+
+        return true;
+    }
 }

app/Services/NetworkTimelineService.php

@@ -77,6 +77,11 @@ class NetworkTimelineService
 		if(self::count() == 0 || $force == true) {
 			$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
 			Redis::del(self::CACHE_KEY);
+            $filteredDomains = collect(InstanceService::getBannedDomains())
+                ->merge(InstanceService::getUnlistedDomains())
+                ->unique()
+                ->values()
+                ->toArray();
 			$ids = Status::whereNotNull('uri')
 				->whereScope('public')
 				->when($hideNsfw, function($q, $hideNsfw) {
@@ -88,7 +93,13 @@ class NetworkTimelineService
 				->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
 				->orderByDesc('created_at')
 				->limit($limit)
-				->pluck('id');
+				->pluck('uri', 'id');
+            $ids = $ids->filter(function($k, $v) use($filteredDomains) {
+                $domain = parse_url($k, PHP_URL_HOST);
+                return !in_array($domain, $filteredDomains);
+            })->map(function($k, $v) {
+                return $v;
+            })->flatten();
 			foreach($ids as $id) {
 				self::add($id);
 			}

app/Util/ActivityPub/Helpers.php

@@ -510,7 +510,14 @@ class Helpers {
 			$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))) &&
 			(config('instance.hide_nsfw_on_public_feeds') == true ? $status->is_nsfw == false : true)
 		) {
+			$filteredDomains = collect(InstanceService::getBannedDomains())
+                ->merge(InstanceService::getUnlistedDomains())
+                ->unique()
+                ->values()
+                ->toArray();
+            if(!in_array($urlDomain, $filteredDomains)) {
 				NetworkTimelineService::add($status->id);
+            }
 		}
 
 		IncrementPostCount::dispatch($pid)->onQueue('low');

routes/api.php

@@ -193,5 +193,9 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
 		Route::get('users/list', 'Api\AdminApiController@getUsers')->middleware($middleware);
 		Route::get('users/get', 'Api\AdminApiController@getUser')->middleware($middleware);
 		Route::post('users/action', 'Api\AdminApiController@userAdminAction')->middleware($middleware);
+		Route::get('instances/list', 'Api\AdminApiController@instances')->middleware($middleware);
+		Route::get('instances/get', 'Api\AdminApiController@getInstance')->middleware($middleware);
+		Route::post('instances/moderate', 'Api\AdminApiController@moderateInstance')->middleware($middleware);
+		Route::post('instances/refresh-stats', 'Api\AdminApiController@refreshInstanceStats')->middleware($middleware);
 	});
 });