Merge pull request #5125 from pixelfed/staging

Add /api/v1/instance/peers API endpoint, disabled by default

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.12.1...dev)
 
+### Added
+- New api/v1/instance/peers API endpoint, disabled by default ([4aad1c22](https://github.com/pixelfed/pixelfed/commit/4aad1c22))
+
 ### Updates
 - Update DirectMessageController, add 72 hour delay for new accounts before they can send a DM ([61d105fd](https://github.com/pixelfed/pixelfed/commit/61d105fd))
 - Update AdminCuratedRegisterController, increase message length from 1000 to 3000 ([9a5e3471](https://github.com/pixelfed/pixelfed/commit/9a5e3471))
@@ -13,6 +16,8 @@
 - Update CollectionsController, add new self route ([bc2495c6](https://github.com/pixelfed/pixelfed/commit/bc2495c6))
 - Update FederationController, add webfinger support for actor uri. Fixes #5068 ([24194f7d](https://github.com/pixelfed/pixelfed/commit/24194f7d))
 - Update FetchNodeinfoPipeline, set last_fetched_at timestamp ([a7fce91e](https://github.com/pixelfed/pixelfed/commit/a7fce91e))
+- Update task scheduler, add weekly instance scan to check nodeinfo for known instances ([dc6b9f46](https://github.com/pixelfed/pixelfed/commit/dc6b9f46))
+-  ([](https://github.com/pixelfed/pixelfed/commit/))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 

app/Console/Commands/WeeklyInstanceScan.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Instance;
+use App\Jobs\InstancePipeline\FetchNodeinfoPipeline;
+use Illuminate\Console\Command;
+
+use function Laravel\Prompts\progress;
+
+class WeeklyInstanceScan extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:weekly-instance-scan';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Scan instance nodeinfo';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        if ((bool) config_cache('federation.activitypub.enabled') == false) {
+            return;
+        }
+
+        $users = progress(
+            label: 'Updating instance stats...',
+            steps: Instance::all(),
+            callback: fn ($instance) => $this->updateInstanceStats($instance),
+        );
+    }
+
+    protected function updateInstanceStats($instance)
+    {
+        FetchNodeinfoPipeline::dispatch($instance)->onQueue('intbg');
+    }
+}

app/Console/Kernel.php

@@ -19,7 +19,6 @@ class Kernel extends ConsoleKernel
     /**
      * Define the application's command schedule.
      *
-     * @param \Illuminate\Console\Scheduling\Schedule $schedule
      *
      * @return void
      */
@@ -32,6 +31,7 @@ class Kernel extends ConsoleKernel
         $schedule->command('gc:failedjobs')->dailyAt(3)->onOneServer();
         $schedule->command('gc:passwordreset')->dailyAt('09:41')->onOneServer();
         $schedule->command('gc:sessions')->twiceDaily(13, 23)->onOneServer();
+        $schedule->command('app:weekly-instance-scan')->weeklyOn(2, '4:20')->onOneServer();
 
         if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) {
             $schedule->command('media:s3gc')->hourlyAt(15);

app/Http/Controllers/Api/ApiV1Controller.php

@@ -4204,4 +4204,26 @@ class ApiV1Controller extends Controller
 
         return $this->json([]);
     }
+
+    /**
+     * GET /api/v1/instance/peers
+     *
+     *
+     * @return array
+     */
+    public function instancePeers(Request $request)
+    {
+        if ((bool) config('instance.show_peers') == false) {
+            return $this->json([]);
+        }
+
+        return $this->json(
+            Cache::remember(InstanceService::CACHE_KEY_API_PEERS_LIST, now()->addHours(24), function () {
+                return Instance::whereNotNull('nodeinfo_last_fetched')
+                    ->whereBanned(false)
+                    ->where('nodeinfo_last_fetched', '>', now()->subDays(8))
+                    ->pluck('domain');
+            })
+        );
+    }
 }

app/Services/InstanceService.php

@@ -2,20 +2,26 @@
 
 namespace App\Services;
 
-use Cache;
 use App\Instance;
 use App\Util\Blurhash\Blurhash;
-use App\Services\ConfigCacheService;
+use Cache;
 
 class InstanceService
 {
     const CACHE_KEY_BY_DOMAIN = 'pf:services:instance:by_domain:';
+
     const CACHE_KEY_BANNED_DOMAINS = 'instances:banned:domains';
+
     const CACHE_KEY_UNLISTED_DOMAINS = 'instances:unlisted:domains';
+
     const CACHE_KEY_NSFW_DOMAINS = 'instances:auto_cw:domains';
+
     const CACHE_KEY_STATS = 'pf:services:instances:stats';
+
     const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
 
+    const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
+
     public function __construct()
     {
         ini_set('memory_limit', config('pixelfed.memory_limit', '1024M'));
@@ -52,11 +58,13 @@ class InstanceService
     public static function software($domain)
     {
         $key = 'instances:software:'.strtolower($domain);
+
         return Cache::remember($key, 86400, function () use ($domain) {
             $instance = Instance::whereDomain($domain)->first();
             if (! $instance) {
                 return;
             }
+
             return $instance->software;
         });
     }
@@ -68,7 +76,7 @@ class InstanceService
                 'total_count' => Instance::count(),
                 'new_count' => Instance::where('created_at', '>', now()->subDays(14))->count(),
                 'banned_count' => Instance::whereBanned(true)->count(),
-				'nsfw_count' => Instance::whereAutoCw(true)->count()
+                'nsfw_count' => Instance::whereAutoCw(true)->count(),
             ];
         });
     }
@@ -79,6 +87,7 @@ class InstanceService
         Cache::forget(self::CACHE_KEY_UNLISTED_DOMAINS);
         Cache::forget(self::CACHE_KEY_NSFW_DOMAINS);
         Cache::forget(self::CACHE_KEY_STATS);
+        Cache::forget(self::CACHE_KEY_API_PEERS_LIST);
 
         self::getBannedDomains();
         self::getUnlistedDomains();
@@ -109,9 +118,9 @@ class InstanceService
             $height = imagesy($image);
 
             $pixels = [];
-			for ($y = 0; $y < $height; ++$y) {
+            for ($y = 0; $y < $height; $y++) {
                 $row = [];
-				for ($x = 0; $x < $width; ++$x) {
+                for ($x = 0; $x < $width; $x++) {
                     $index = imagecolorat($image, $x, $y);
                     $colors = imagecolorsforindex($image, $index);
 

config/instance.php

@@ -16,7 +16,7 @@ return [
             'enabled' => env('EXP_LOOPS', false),
         ],
         'tags' => [
-			'is_public' => env('INSTANCE_PUBLIC_HASHTAGS', false)
+            'is_public' => env('INSTANCE_PUBLIC_HASHTAGS', false),
         ],
     ],
 
@@ -25,30 +25,30 @@ return [
     'timeline' => [
         'home' => [
             'cached' => env('PF_HOME_TIMELINE_CACHE', false),
-			'cache_ttl' => env('PF_HOME_TIMELINE_CACHE_TTL', 900)
+            'cache_ttl' => env('PF_HOME_TIMELINE_CACHE_TTL', 900),
         ],
 
         'local' => [
             'cached' => env('INSTANCE_PUBLIC_TIMELINE_CACHED', false),
-			'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
+            'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false),
         ],
 
         'network' => [
             'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
             'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
-			'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
+            'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6),
-		]
+        ],
     ],
 
     'page' => [
         '404' => [
             'header' => env('PAGE_404_HEADER', 'Sorry, this page isn\'t available.'),
-			'body' => env('PAGE_404_BODY', 'The link you followed may be broken, or the page may have been removed. <a href="/">Go back to Pixelfed.</a>')
+            'body' => env('PAGE_404_BODY', 'The link you followed may be broken, or the page may have been removed. <a href="/">Go back to Pixelfed.</a>'),
         ],
         '503' => [
             'header' => env('PAGE_503_HEADER', 'Service Unavailable'),
-			'body' => env('PAGE_503_BODY', 'Our service is in maintenance mode, please try again later.')
+            'body' => env('PAGE_503_BODY', 'Our service is in maintenance mode, please try again later.'),
-		]
+        ],
     ],
 
     'username' => [
@@ -56,12 +56,12 @@ return [
         'remote' => [
             'formats' => ['@', 'from', 'custom'],
             'format' => in_array(env('USERNAME_REMOTE_FORMAT', '@'), ['@', 'from', 'custom']) ? env('USERNAME_REMOTE_FORMAT', '@') : '@',
-			'custom' => env('USERNAME_REMOTE_CUSTOM_TEXT', null)
+            'custom' => env('USERNAME_REMOTE_CUSTOM_TEXT', null),
-		]
+        ],
     ],
 
     'polls' => [
-		'enabled' => false
+        'enabled' => false,
     ],
 
     'stories' => [
@@ -70,7 +70,7 @@ return [
 
     'restricted' => [
         'enabled' => env('RESTRICTED_INSTANCE', false),
-		'level' => 1
+        'level' => 1,
     ],
 
     'oauth' => [
@@ -79,15 +79,15 @@ return [
         'pat' => [
             'enabled' => env('OAUTH_PAT_ENABLED', false),
             'id' => env('OAUTH_PAT_ID'),
-		]
+        ],
     ],
 
     'label' => [
         'covid' => [
             'enabled' => env('ENABLE_COVID_LABEL', true),
             'url' => env('COVID_LABEL_URL', 'https://www.who.int/emergencies/diseases/novel-coronavirus-2019/advice-for-public'),
-			'org' => env('COVID_LABEL_ORG', 'visit the WHO website')
+            'org' => env('COVID_LABEL_ORG', 'visit the WHO website'),
-		]
+        ],
     ],
 
     'enable_cc' => env('ENABLE_CONFIG_CACHE', true),
@@ -102,11 +102,11 @@ return [
     'hide_nsfw_on_public_feeds' => env('PF_HIDE_NSFW_ON_PUBLIC_FEEDS', false),
 
     'avatar' => [
-		'local_to_cloud' => env('PF_LOCAL_AVATAR_TO_CLOUD', false)
+        'local_to_cloud' => env('PF_LOCAL_AVATAR_TO_CLOUD', false),
     ],
 
     'admin_invites' => [
-		'enabled' => env('PF_ADMIN_INVITES_ENABLED', true)
+        'enabled' => env('PF_ADMIN_INVITES_ENABLED', true),
     ],
 
     'user_filters' => [
@@ -119,8 +119,8 @@ return [
         'email' => [
             'enabled' => env('INSTANCE_REPORTS_EMAIL_ENABLED', false),
             'to' => env('INSTANCE_REPORTS_EMAIL_ADDRESSES'),
-			'autospam' => env('INSTANCE_REPORTS_EMAIL_AUTOSPAM', false)
+            'autospam' => env('INSTANCE_REPORTS_EMAIL_AUTOSPAM', false),
-		]
+        ],
     ],
 
     'landing' => [
@@ -129,7 +129,7 @@ return [
     ],
 
     'banner' => [
-		'blurhash' => env('INSTANCE_BANNER_BLURHASH', 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt')
+        'blurhash' => env('INSTANCE_BANNER_BLURHASH', 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt'),
     ],
 
     'parental_controls' => [
@@ -143,14 +143,14 @@ return [
     ],
 
     'software-update' => [
-        'disable_failed_warning' => env('INSTANCE_SOFTWARE_UPDATE_DISABLE_FAILED_WARNING', false)
+        'disable_failed_warning' => env('INSTANCE_SOFTWARE_UPDATE_DISABLE_FAILED_WARNING', false),
     ],
 
     'notifications' => [
         'gc' => [
             'enabled' => env('INSTANCE_NOTIFY_AUTO_GC', false),
-            'delete_after_days' => env('INSTANCE_NOTIFY_AUTO_GC_DEL_AFTER_DAYS', 365)
+            'delete_after_days' => env('INSTANCE_NOTIFY_AUTO_GC_DEL_AFTER_DAYS', 365),
-        ]
+        ],
     ],
 
     'curated_registration' => [
@@ -173,7 +173,9 @@ return [
                     'max_per_day' => env('INSTANCE_CUR_REG_NOTIFY_ADMIN_ON_VERIFY_MPD', 10),
                 ],
                 'on_user_response' => env('INSTANCE_CUR_REG_NOTIFY_ADMIN_ON_USER_RESPONSE', false),
-            ]
             ],
         ],
+    ],
+
+    'show_peers' => env('INSTANCE_SHOW_PEERS', false),
 ];

database/migrations/2024_05_31_090555_update_instances_table_add_index_to_nodeinfo_last_fetched_at.php

@@ -0,0 +1,36 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('instances', function (Blueprint $table) {
+            $schemaManager = Schema::getConnection()->getDoctrineSchemaManager();
+            $indexesFound = $schemaManager->listTableIndexes('instances');
+            if (! array_key_exists('instances_nodeinfo_last_fetched_index', $indexesFound)) {
+                $table->index('nodeinfo_last_fetched');
+            }
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('instances', function (Blueprint $table) {
+            $schemaManager = Schema::getConnection()->getDoctrineSchemaManager();
+            $indexesFound = $schemaManager->listTableIndexes('instances');
+            if (array_key_exists('instances_nodeinfo_last_fetched_index', $indexesFound)) {
+                $table->dropIndex('instances_nodeinfo_last_fetched_index');
+            }
+        });
+    }
+};

routes/api.php

@@ -26,6 +26,7 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
         Route::post('apps', 'Api\ApiV1Controller@apps');
         Route::get('apps/verify_credentials', 'Api\ApiV1Controller@getApp')->middleware($middleware);
         Route::get('instance', 'Api\ApiV1Controller@instance');
+        Route::get('instance/peers', 'Api\ApiV1Controller@instancePeers');
         Route::get('bookmarks', 'Api\ApiV1Controller@bookmarks')->middleware($middleware);
 
         Route::get('accounts/verify_credentials', 'Api\ApiV1Controller@verifyCredentials')->middleware($middleware);