Merge pull request #4902 from pixelfed/staging

Staging
2024-11-25 15:55:22 +00:00 · 2024-02-07 04:45:47 -07:00 · 2024-02-07 04:45:47 -07:00 · 6ea20716bc
commit 6ea20716bc
parent 111ba70473 1f3f0cae65
8 changed files with 691 additions and 268 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -98,6 +98,12 @@
 - Update AP helpers, fix sensitive bug ([00ed330c](https://github.com/pixelfed/pixelfed/commit/00ed330c))
 - Update NotificationEpochUpdatePipeline, use more efficient query ([4d401389](https://github.com/pixelfed/pixelfed/commit/4d401389))
 - Update notification pipelines, fix non-local saving ([fa97a1f3](https://github.com/pixelfed/pixelfed/commit/fa97a1f3))
+- Update NodeinfoService, disable redirects ([240e6bbe](https://github.com/pixelfed/pixelfed/commit/240e6bbe))
+- Update Instance model, add entity casts ([289cad47](https://github.com/pixelfed/pixelfed/commit/289cad47))
+- Update FetchNodeinfoPipeline, use more efficient dispatch ([ac01f51a](https://github.com/pixelfed/pixelfed/commit/ac01f51a))
+- Update horizon.php config ([1e3acade](https://github.com/pixelfed/pixelfed/commit/1e3acade))
+- Update PublicApiController, consume InstanceService blocked domains for account and statuses endpoints ([01b33fb3](https://github.com/pixelfed/pixelfed/commit/01b33fb3))
+- Update ApiV1Controller, enforce blocked instance domain logic ([5b284cac](https://github.com/pixelfed/pixelfed/commit/5b284cac))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))

 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)
--- a/app/Console/Commands/InstanceManager.php
+++ b/app/Console/Commands/InstanceManager.php
@ -0,0 +1,298 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Instance;
+use App\Profile;
+use App\Services\InstanceService;
+use App\Jobs\InstancePipeline\FetchNodeinfoPipeline;
+use function Laravel\Prompts\select;
+use function Laravel\Prompts\confirm;
+use function Laravel\Prompts\progress;
+use function Laravel\Prompts\search;
+use function Laravel\Prompts\table;
+
+class InstanceManager extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:instance-manager';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Manage Instances';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $action = select(
+            'What action do you want to perform?',
+            [
+                'Recalculate Stats',
+                'Ban Instance',
+                'Unlist Instance',
+                'Unlisted Instances',
+                'Banned Instances',
+                'Unban Instance',
+                'Relist Instance',
+            ],
+        );
+
+        switch($action) {
+            case 'Recalculate Stats':
+                return $this->recalculateStats();
+            break;
+
+            case 'Unlisted Instances':
+                return $this->viewUnlistedInstances();
+            break;
+
+            case 'Banned Instances':
+                return $this->viewBannedInstances();
+            break;
+
+            case 'Unlist Instance':
+                return $this->unlistInstance();
+            break;
+
+            case 'Ban Instance':
+                return $this->banInstance();
+            break;
+
+            case 'Unban Instance':
+                return $this->unbanInstance();
+            break;
+
+            case 'Relist Instance':
+                return $this->relistInstance();
+            break;
+        }
+    }
+
+    protected function recalculateStats()
+    {
+        $instanceCount = Instance::count();
+        $confirmed = confirm('Do you want to recalculate stats for all ' . $instanceCount . ' instances?');
+        if(!$confirmed) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        $users = progress(
+            label: 'Updating instance stats...',
+            steps: Instance::all(),
+            callback: fn ($instance) => $this->updateInstanceStats($instance),
+        );
+    }
+
+    protected function updateInstanceStats($instance)
+    {
+        FetchNodeinfoPipeline::dispatch($instance)->onQueue('intbg');
+    }
+
+    protected function unlistInstance()
+    {
+        $id = search(
+            'Search by domain',
+            fn (string $value) => strlen($value) > 0
+                ? Instance::whereUnlisted(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all()
+                : []
+        );
+
+        $instance = Instance::find($id);
+        if(!$instance) {
+            $this->error('Oops, an error occured');
+            exit;
+        }
+
+        $tbl = [
+            [
+                $instance->domain,
+                number_format($instance->status_count),
+                number_format($instance->user_count),
+            ]
+        ];
+        table(
+            ['Domain', 'Status Count', 'User Count'],
+            $tbl
+        );
+
+        $confirmed = confirm('Are you sure you want to unlist this instance?');
+        if(!$confirmed) {
+            $this->error('Aborting instance unlisting');
+            exit;
+        }
+
+        $instance->unlisted = true;
+        $instance->save();
+        InstanceService::refresh();
+        $this->info('Successfully unlisted ' . $instance->domain . '!');
+        exit;
+    }
+
+    protected function relistInstance()
+    {
+        $id = search(
+            'Search by domain',
+            fn (string $value) => strlen($value) > 0
+                ? Instance::whereUnlisted(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all()
+                : []
+        );
+
+        $instance = Instance::find($id);
+        if(!$instance) {
+            $this->error('Oops, an error occured');
+            exit;
+        }
+
+        $tbl = [
+            [
+                $instance->domain,
+                number_format($instance->status_count),
+                number_format($instance->user_count),
+            ]
+        ];
+        table(
+            ['Domain', 'Status Count', 'User Count'],
+            $tbl
+        );
+
+        $confirmed = confirm('Are you sure you want to re-list this instance?');
+        if(!$confirmed) {
+            $this->error('Aborting instance re-listing');
+            exit;
+        }
+
+        $instance->unlisted = false;
+        $instance->save();
+        InstanceService::refresh();
+        $this->info('Successfully re-listed ' . $instance->domain . '!');
+        exit;
+    }
+
+    protected function banInstance()
+    {
+        $id = search(
+            'Search by domain',
+            fn (string $value) => strlen($value) > 0
+                ? Instance::whereBanned(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all()
+                : []
+        );
+
+        $instance = Instance::find($id);
+        if(!$instance) {
+            $this->error('Oops, an error occured');
+            exit;
+        }
+
+        $tbl = [
+            [
+                $instance->domain,
+                number_format($instance->status_count),
+                number_format($instance->user_count),
+            ]
+        ];
+        table(
+            ['Domain', 'Status Count', 'User Count'],
+            $tbl
+        );
+
+        $confirmed = confirm('Are you sure you want to ban this instance?');
+        if(!$confirmed) {
+            $this->error('Aborting instance ban');
+            exit;
+        }
+
+        $instance->banned = true;
+        $instance->save();
+        InstanceService::refresh();
+        $this->info('Successfully banned ' . $instance->domain . '!');
+        exit;
+    }
+
+    protected function unbanInstance()
+    {
+        $id = search(
+            'Search by domain',
+            fn (string $value) => strlen($value) > 0
+                ? Instance::whereBanned(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all()
+                : []
+        );
+
+        $instance = Instance::find($id);
+        if(!$instance) {
+            $this->error('Oops, an error occured');
+            exit;
+        }
+
+        $tbl = [
+            [
+                $instance->domain,
+                number_format($instance->status_count),
+                number_format($instance->user_count),
+            ]
+        ];
+        table(
+            ['Domain', 'Status Count', 'User Count'],
+            $tbl
+        );
+
+        $confirmed = confirm('Are you sure you want to unban this instance?');
+        if(!$confirmed) {
+            $this->error('Aborting instance unban');
+            exit;
+        }
+
+        $instance->banned = false;
+        $instance->save();
+        InstanceService::refresh();
+        $this->info('Successfully un-banned ' . $instance->domain . '!');
+        exit;
+    }
+
+    protected function viewBannedInstances()
+    {
+        $data = Instance::whereBanned(true)
+            ->get(['domain', 'user_count', 'status_count'])
+            ->map(function($d) {
+                return [
+                    'domain' => $d->domain,
+                    'user_count' => number_format($d->user_count),
+                    'status_count' => number_format($d->status_count),
+                ];
+            })
+            ->toArray();
+        table(
+            ['Domain', 'User Count', 'Status Count'],
+            $data
+        );
+    }
+
+    protected function viewUnlistedInstances()
+    {
+        $data = Instance::whereUnlisted(true)
+            ->get(['domain', 'user_count', 'status_count', 'banned'])
+            ->map(function($d) {
+                return [
+                    'domain' => $d->domain,
+                    'user_count' => number_format($d->user_count),
+                    'status_count' => number_format($d->status_count),
+                    'banned' => $d->banned ? '✅' : null
+                ];
+            })
+            ->toArray();
+        table(
+            ['Domain', 'User Count', 'Status Count', 'Banned'],
+            $data
+        );
+    }
+}
--- a/app/Http/Controllers/Api/ApiV1Controller.php
+++ b/app/Http/Controllers/Api/ApiV1Controller.php
@ -219,6 +219,10 @@ class ApiV1Controller extends Controller
        if(!$res) {
            return response()->json(['error' => 'Record not found'], 404);
        }
+        if($res && strpos($res['acct'], '@') != -1) {
+            $domain = parse_url($res['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
        return $this->json($res);
    }

@ -483,6 +487,11 @@ class ApiV1Controller extends Controller
        $limit = $request->input('limit', 10);
        $napi = $request->has(self::PF_API_ENTITY_KEY);

+        if($account && strpos($account['acct'], '@') != -1) {
+            $domain = parse_url($account['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        if(intval($pid) !== intval($account['id'])) {
            if($account['locked']) {
                if(!FollowerService::follows($pid, $account['id'])) {
@ -575,6 +584,11 @@ class ApiV1Controller extends Controller
        $limit = $request->input('limit', 10);
        $napi = $request->has(self::PF_API_ENTITY_KEY);

+        if($account && strpos($account['acct'], '@') != -1) {
+            $domain = parse_url($account['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        if(intval($pid) !== intval($account['id'])) {
            if($account['locked']) {
                if(!FollowerService::follows($pid, $account['id'])) {
@ -676,6 +690,11 @@ class ApiV1Controller extends Controller
            return $this->json(['error' => 'Account not found'], 404);
        }

+        if($profile && strpos($profile['acct'], '@') != -1) {
+            $domain = parse_url($profile['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $limit = $request->limit ?? 20;
        $max_id = $request->max_id;
        $min_id = $request->min_id;
@ -766,6 +785,11 @@ class ApiV1Controller extends Controller
            ->whereNull('status')
            ->findOrFail($id);

+        if($target && $target->domain) {
+            $domain = $target->domain;
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $private = (bool) $target->is_private;
        $remote = (bool) $target->domain;
        $blocked = UserFilter::whereUserId($target->id)
@ -1252,14 +1276,19 @@ class ApiV1Controller extends Controller
        $user = $request->user();
        abort_if($user->has_roles && !UserRoleService::can('can-like', $user->id), 403, 'Invalid permissions for this action');

-        AccountService::setLastActive($user->id);
-
        $status = StatusService::getMastodon($id, false);

-        abort_unless($status, 400);
+        abort_unless($status, 404);
+
+        if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) {
+            $domain = parse_url($status['account']['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }

        $spid = $status['account']['id'];

+        AccountService::setLastActive($user->id);
+
        if(intval($spid) !== intval($user->profile_id)) {
            if($status['visibility'] == 'private') {
                abort_if(!FollowerService::follows($user->profile_id, $spid), 403);
@ -1404,6 +1433,11 @@ class ApiV1Controller extends Controller
            return response()->json(['error' => 'Record not found'], 404);
        }

+        if($target && strpos($target['acct'], '@') != -1) {
+            $domain = parse_url($target['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $followRequest = FollowRequest::whereFollowingId($pid)->whereFollowerId($id)->first();

        if(!$followRequest) {
@ -2011,6 +2045,11 @@ class ApiV1Controller extends Controller

        $account = Profile::findOrFail($id);

+        if($account && $account->domain) {
+            $domain = $account->domain;
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $count = UserFilterService::muteCount($pid);
        $maxLimit = intval(config('instance.user_filters.max_user_mutes'));
        if($count == 0) {
@ -2653,6 +2692,11 @@ class ApiV1Controller extends Controller
            abort(404);
        }

+        if($res && isset($res['account'], $res['account']['acct'], $res['account']['url']) && strpos($res['account']['acct'], '@') != -1) {
+            $domain = parse_url($res['account']['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $scope = $res['visibility'];
        if(!in_array($scope, ['public', 'unlisted'])) {
            if($scope === 'private') {
@ -2697,6 +2741,11 @@ class ApiV1Controller extends Controller
            return response('', 404);
        }

+        if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) {
+            $domain = parse_url($status['account']['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        if(intval($status['account']['id']) !== intval($user->profile_id)) {
            if($status['visibility'] == 'private') {
                if(!FollowerService::follows($user->profile_id, $status['account']['id'])) {
@ -2780,6 +2829,10 @@ class ApiV1Controller extends Controller
        $status = Status::findOrFail($id);
        $account = AccountService::get($status->profile_id, true);
        abort_if(!$account, 404);
+        if($account && strpos($account['acct'], '@') != -1) {
+            $domain = parse_url($account['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
        $author = intval($status->profile_id) === intval($pid) || $user->is_admin;
        $napi = $request->has(self::PF_API_ENTITY_KEY);

@ -2871,6 +2924,10 @@ class ApiV1Controller extends Controller
        $pid = $user->profile_id;
        $status = Status::findOrFail($id);
        $account = AccountService::get($status->profile_id, true);
+        if($account && strpos($account['acct'], '@') != -1) {
+            $domain = parse_url($account['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
        abort_if(!$account, 404);
        $author = intval($status->profile_id) === intval($pid) || $user->is_admin;
        $napi = $request->has(self::PF_API_ENTITY_KEY);
@ -3200,7 +3257,11 @@ class ApiV1Controller extends Controller
        abort_if($user->has_roles && !UserRoleService::can('can-share', $user->id), 403, 'Invalid permissions for this action');
        AccountService::setLastActive($user->id);
        $status = Status::whereScope('public')->findOrFail($id);
-
+        if($status && ($status->uri || $status->url || $status->object_url)) {
+            $url = $status->uri ?? $status->url ?? $status->object_url;
+            $domain = parse_url($url, PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
        if(intval($status->profile_id) !== intval($user->profile_id)) {
            if($status->scope == 'private') {
                abort_if(!FollowerService::follows($user->profile_id, $status->profile_id), 403);
--- a/app/Http/Controllers/PublicApiController.php
+++ b/app/Http/Controllers/PublicApiController.php
@ -42,6 +42,7 @@ use App\Services\{
 use App\Jobs\StatusPipeline\NewStatusPipeline;
 use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
+use App\Services\InstanceService;

 class PublicApiController extends Controller
 {
@ -661,6 +662,10 @@ class PublicApiController extends Controller
    public function account(Request $request, $id)
    {
        $res = AccountService::get($id);
+        if($res && isset($res['local'], $res['url']) && !$res['local']) {
+            $domain = parse_url($res['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
        return response()->json($res);
    }

@ -680,6 +685,11 @@ class PublicApiController extends Controller
        $profile = AccountService::get($id);
        abort_if(!$profile, 404);

+        if($profile && isset($profile['local'], $profile['url']) && !$profile['local']) {
+            $domain = parse_url($profile['url'], PHP_URL_HOST);
+            abort_if(in_array($domain, InstanceService::getBannedDomains()), 404);
+        }
+
        $limit = $request->limit ?? 9;
        $max_id = $request->max_id;
        $min_id = $request->min_id;
--- a/app/Instance.php
+++ b/app/Instance.php
@ -6,7 +6,21 @@ use Illuminate\Database\Eloquent\Model;

 class Instance extends Model
 {
-	protected $fillable = ['domain', 'banned', 'auto_cw', 'unlisted', 'notes'];
+    protected $casts = [
+        'last_crawled_at' => 'datetime',
+        'actors_last_synced_at' => 'datetime',
+        'notes' => 'array',
+        'nodeinfo_last_fetched' => 'datetime',
+        'delivery_next_after' => 'datetime',
+    ];
+
+    protected $fillable = [
+        'domain',
+        'banned',
+        'auto_cw',
+        'unlisted',
+        'notes'
+    ];

    public function profiles()
    {
--- a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
+++ b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
@ -4,6 +4,7 @@ namespace App\Jobs\InstancePipeline;

 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
@ -12,8 +13,10 @@ use Illuminate\Support\Facades\Http;
 use App\Instance;
 use App\Profile;
 use App\Services\NodeinfoService;
+use Illuminate\Contracts\Cache\Repository;
+use Illuminate\Support\Facades\Cache;

-class FetchNodeinfoPipeline implements ShouldQueue
+class FetchNodeinfoPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

@ -29,6 +32,21 @@ class FetchNodeinfoPipeline implements ShouldQueue
        $this->instance = $instance;
    }

+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 14400;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return $this->instance->id;
+    }
+
    /**
     * Execute the job.
     *
@ -38,18 +56,27 @@ class FetchNodeinfoPipeline implements ShouldQueue
    {
        $instance = $this->instance;

+        if( $instance->nodeinfo_last_fetched &&
+            $instance->nodeinfo_last_fetched->gt(now()->subHours(12)) ||
+            $instance->delivery_timeout &&
+            $instance->delivery_next_after->gt(now())
+        ) {
+            return;
+        }
+
        $ni = NodeinfoService::get($instance->domain);
+        $instance->last_crawled_at = now();
        if($ni) {
            if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) {
                $software = $ni['software']['name'];
                $instance->software = strtolower(strip_tags($software));
-				$instance->last_crawled_at = now();
                $instance->user_count = Profile::whereDomain($instance->domain)->count();
+                $instance->nodeinfo_last_fetched = now();
                $instance->save();
            }
        } else {
-			$instance->user_count = Profile::whereDomain($instance->domain)->count();
-			$instance->last_crawled_at = now();
+            $instance->delivery_timeout = 1;
+            $instance->delivery_next_after = now()->addHours(14);
            $instance->save();
        }
    }
--- a/app/Services/NodeinfoService.php
+++ b/app/Services/NodeinfoService.php
@ -22,7 +22,10 @@ class NodeinfoService
        $wk = $url . '/.well-known/nodeinfo';

        try {
-            $res = Http::withHeaders($headers)
+            $res = Http::withOptions([
+                'allow_redirects' => false,
+            ])
+            ->withHeaders($headers)
            ->timeout(5)
            ->get($wk);
        } catch (RequestException $e) {
@ -61,7 +64,10 @@ class NodeinfoService
        }

        try {
-            $res = Http::withHeaders($headers)
+            $res = Http::withOptions([
+                'allow_redirects' => false,
+            ])
+            ->withHeaders($headers)
            ->timeout(5)
            ->get($href);
        } catch (RequestException $e) {
--- a/config/horizon.php
+++ b/config/horizon.php
@ -89,6 +89,7 @@ return [
        'redis:delete' => 30,
        'redis:story' => 30,
        'redis:mmo' => 30,
+        'redis:intbg' => 30,
    ],

    /*
@ -172,7 +173,7 @@ return [
        'production' => [
            'supervisor-1' => [
                'connection'    => 'redis',
-				'queue'         => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'],
+                'queue'         => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'],
                'balance'       => env('HORIZON_BALANCE_STRATEGY', 'auto'),
                'minProcesses'  => env('HORIZON_MIN_PROCESSES', 1),
                'maxProcesses'  => env('HORIZON_MAX_PROCESSES', 20),
@ -186,7 +187,7 @@ return [
        'local' => [
            'supervisor-1' => [
                'connection'    => 'redis',
-				'queue'         => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'],
+                'queue'         => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'],
                'balance'       => 'auto',
                'minProcesses' => 1,
                'maxProcesses'  => 20,