Update job queue, separate deletes into their own queue

This commit is contained in:
Daniel Supernault 2021-07-13 23:09:50 -06:00
parent 3b071e56ac
commit 7f4213924f
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
3 changed files with 309 additions and 283 deletions

View file

@ -3,16 +3,17 @@
namespace App\Http\Controllers; namespace App\Http\Controllers;
use App\Jobs\InboxPipeline\{ use App\Jobs\InboxPipeline\{
InboxWorker, DeleteWorker,
InboxValidator InboxWorker,
InboxValidator
}; };
use App\Jobs\RemoteFollowPipeline\RemoteFollowPipeline; use App\Jobs\RemoteFollowPipeline\RemoteFollowPipeline;
use App\{ use App\{
AccountLog, AccountLog,
Like, Like,
Profile, Profile,
Status, Status,
User User
}; };
use App\Util\Lexer\Nickname; use App\Util\Lexer\Nickname;
use App\Util\Webfinger\Webfinger; use App\Util\Webfinger\Webfinger;
@ -23,146 +24,158 @@ use Illuminate\Http\Request;
use League\Fractal; use League\Fractal;
use App\Util\Site\Nodeinfo; use App\Util\Site\Nodeinfo;
use App\Util\ActivityPub\{ use App\Util\ActivityPub\{
Helpers, Helpers,
HttpSignature, HttpSignature,
Outbox Outbox
}; };
use Zttp\Zttp; use Zttp\Zttp;
class FederationController extends Controller class FederationController extends Controller
{ {
public function nodeinfoWellKnown() public function nodeinfoWellKnown()
{ {
abort_if(!config('federation.nodeinfo.enabled'), 404); abort_if(!config('federation.nodeinfo.enabled'), 404);
return response()->json(Nodeinfo::wellKnown()) return response()->json(Nodeinfo::wellKnown())
->header('Access-Control-Allow-Origin','*'); ->header('Access-Control-Allow-Origin','*');
} }
public function nodeinfo() public function nodeinfo()
{ {
abort_if(!config('federation.nodeinfo.enabled'), 404); abort_if(!config('federation.nodeinfo.enabled'), 404);
return response()->json(Nodeinfo::get()) return response()->json(Nodeinfo::get())
->header('Access-Control-Allow-Origin','*'); ->header('Access-Control-Allow-Origin','*');
} }
public function webfinger(Request $request) public function webfinger(Request $request)
{ {
abort_if(!config('federation.webfinger.enabled'), 400); abort_if(!config('federation.webfinger.enabled'), 400);
abort_if(!$request->filled('resource'), 400); abort_if(!$request->filled('resource'), 400);
$resource = $request->input('resource'); $resource = $request->input('resource');
$parsed = Nickname::normalizeProfileUrl($resource); $parsed = Nickname::normalizeProfileUrl($resource);
if(empty($parsed) || $parsed['domain'] !== config('pixelfed.domain.app')) { if(empty($parsed) || $parsed['domain'] !== config('pixelfed.domain.app')) {
abort(404); abort(404);
} }
$username = $parsed['username']; $username = $parsed['username'];
$profile = Profile::whereNull('domain')->whereUsername($username)->firstOrFail(); $profile = Profile::whereNull('domain')->whereUsername($username)->firstOrFail();
if($profile->status != null) { if($profile->status != null) {
return ProfileController::accountCheck($profile); return ProfileController::accountCheck($profile);
} }
$webfinger = (new Webfinger($profile))->generate(); $webfinger = (new Webfinger($profile))->generate();
return response()->json($webfinger, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES) return response()->json($webfinger, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES)
->header('Access-Control-Allow-Origin','*'); ->header('Access-Control-Allow-Origin','*');
} }
public function hostMeta(Request $request) public function hostMeta(Request $request)
{ {
abort_if(!config('federation.webfinger.enabled'), 404); abort_if(!config('federation.webfinger.enabled'), 404);
$path = route('well-known.webfinger'); $path = route('well-known.webfinger');
$xml = '<?xml version="1.0" encoding="UTF-8"?><XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="'.$path.'?resource={uri}"/></XRD>'; $xml = '<?xml version="1.0" encoding="UTF-8"?><XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="'.$path.'?resource={uri}"/></XRD>';
return response($xml)->header('Content-Type', 'application/xrd+xml'); return response($xml)->header('Content-Type', 'application/xrd+xml');
} }
public function userOutbox(Request $request, $username) public function userOutbox(Request $request, $username)
{ {
abort_if(!config_cache('federation.activitypub.enabled'), 404); abort_if(!config_cache('federation.activitypub.enabled'), 404);
abort_if(!config('federation.activitypub.outbox'), 404); abort_if(!config('federation.activitypub.outbox'), 404);
$profile = Profile::whereNull('domain') $profile = Profile::whereNull('domain')
->whereNull('status') ->whereNull('status')
->whereIsPrivate(false) ->whereIsPrivate(false)
->whereUsername($username) ->whereUsername($username)
->firstOrFail(); ->firstOrFail();
$key = 'ap:outbox:latest_10:pid:' . $profile->id; $key = 'ap:outbox:latest_10:pid:' . $profile->id;
$ttl = now()->addMinutes(15); $ttl = now()->addMinutes(15);
$res = Cache::remember($key, $ttl, function() use($profile) { $res = Cache::remember($key, $ttl, function() use($profile) {
return Outbox::get($profile); return Outbox::get($profile);
}); });
return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json'); return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json');
} }
public function userInbox(Request $request, $username) public function userInbox(Request $request, $username)
{ {
abort_if(!config_cache('federation.activitypub.enabled'), 404); abort_if(!config_cache('federation.activitypub.enabled'), 404);
abort_if(!config('federation.activitypub.inbox'), 404); abort_if(!config('federation.activitypub.inbox'), 404);
$headers = $request->headers->all(); $headers = $request->headers->all();
$payload = $request->getContent(); $payload = $request->getContent();
dispatch(new InboxValidator($username, $headers, $payload))->onQueue('high'); $obj = json_decode($payload, true, 8);
return;
}
public function sharedInbox(Request $request) if(isset($obj['type']) && $obj['type'] === 'Delete') {
{ dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
abort_if(!config_cache('federation.activitypub.enabled'), 404); } else {
abort_if(!config('federation.activitypub.sharedInbox'), 404); dispatch(new InboxValidator($username, $headers, $payload))->onQueue('high');
}
return;
}
$headers = $request->headers->all(); public function sharedInbox(Request $request)
$payload = $request->getContent(); {
dispatch(new InboxWorker($headers, $payload))->onQueue('high'); abort_if(!config_cache('federation.activitypub.enabled'), 404);
return; abort_if(!config('federation.activitypub.sharedInbox'), 404);
}
public function userFollowing(Request $request, $username) $headers = $request->headers->all();
{ $payload = $request->getContent();
abort_if(!config_cache('federation.activitypub.enabled'), 404); $obj = json_decode($payload, true, 8);
$profile = Profile::whereNull('remote_url') if(isset($obj['type']) && $obj['type'] === 'Delete') {
->whereUsername($username) dispatch(new DeleteWorker($headers, $payload))->onQueue('delete');
->whereIsPrivate(false) } else {
->firstOrFail(); dispatch(new InboxWorker($headers, $payload))->onQueue('high');
}
return;
}
if($profile->status != null) { public function userFollowing(Request $request, $username)
abort(404); {
} abort_if(!config_cache('federation.activitypub.enabled'), 404);
$obj = [ $profile = Profile::whereNull('remote_url')
'@context' => 'https://www.w3.org/ns/activitystreams', ->whereUsername($username)
'id' => $request->getUri(), ->whereIsPrivate(false)
'type' => 'OrderedCollectionPage', ->firstOrFail();
'totalItems' => 0,
'orderedItems' => []
];
return response()->json($obj);
}
public function userFollowers(Request $request, $username) if($profile->status != null) {
{ abort(404);
abort_if(!config_cache('federation.activitypub.enabled'), 404); }
$profile = Profile::whereNull('remote_url') $obj = [
->whereUsername($username) '@context' => 'https://www.w3.org/ns/activitystreams',
->whereIsPrivate(false) 'id' => $request->getUri(),
->firstOrFail(); 'type' => 'OrderedCollectionPage',
'totalItems' => 0,
'orderedItems' => []
];
return response()->json($obj);
}
if($profile->status != null) { public function userFollowers(Request $request, $username)
abort(404); {
} abort_if(!config_cache('federation.activitypub.enabled'), 404);
$obj = [ $profile = Profile::whereNull('remote_url')
'@context' => 'https://www.w3.org/ns/activitystreams', ->whereUsername($username)
'id' => $request->getUri(), ->whereIsPrivate(false)
'type' => 'OrderedCollectionPage', ->firstOrFail();
'totalItems' => 0,
'orderedItems' => []
];
return response()->json($obj); if($profile->status != null) {
} abort(404);
}
$obj = [
'@context' => 'https://www.w3.org/ns/activitystreams',
'id' => $request->getUri(),
'type' => 'OrderedCollectionPage',
'totalItems' => 0,
'orderedItems' => []
];
return response()->json($obj);
}
} }

View file

@ -50,7 +50,7 @@ class DeleteWorker implements ShouldQueue
$payload = json_decode($this->payload, true, 8); $payload = json_decode($this->payload, true, 8);
if(isset($payload['id'])) { if(isset($payload['id'])) {
$lockKey = hash('sha256', $payload['id']); $lockKey = 'pf:ap:del-lock:' . hash('sha256', $payload['id']);
if(Cache::get($lockKey) !== null) { if(Cache::get($lockKey) !== null) {
// Job processed already // Job processed already
return 1; return 1;
@ -116,6 +116,18 @@ class DeleteWorker implements ShouldQueue
return; return;
} }
$profile = null;
if($this->verifySignature($headers, $payload) == true) {
(new Inbox($headers, $profile, $payload))->handle();
return;
} else if($this->blindKeyRotation($headers, $payload) == true) {
(new Inbox($headers, $profile, $payload))->handle();
return;
} else {
return;
}
} }
protected function verifySignature($headers, $payload) protected function verifySignature($headers, $payload)

View file

@ -2,190 +2,191 @@
return [ return [
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Horizon Domain | Horizon Domain
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This is the subdomain where Horizon will be accessible from. If this | This is the subdomain where Horizon will be accessible from. If this
| setting is null, Horizon will reside under the same domain as the | setting is null, Horizon will reside under the same domain as the
| application. Otherwise, this value will serve as the subdomain. | application. Otherwise, this value will serve as the subdomain.
| |
*/ */
'domain' => null, 'domain' => null,
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Horizon Path | Horizon Path
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This is the URI path where Horizon will be accessible from. Feel free | This is the URI path where Horizon will be accessible from. Feel free
| to change this path to anything you like. Note that the URI will not | to change this path to anything you like. Note that the URI will not
| affect the paths of its internal API that aren't exposed to users. | affect the paths of its internal API that aren't exposed to users.
| |
*/ */
'path' => 'horizon', 'path' => 'horizon',
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Horizon Redis Connection | Horizon Redis Connection
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This is the name of the Redis connection where Horizon will store the | This is the name of the Redis connection where Horizon will store the
| meta information required for it to function. It includes the list | meta information required for it to function. It includes the list
| of supervisors, failed jobs, job metrics, and other information. | of supervisors, failed jobs, job metrics, and other information.
| |
*/ */
'use' => 'default', 'use' => 'default',
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Horizon Redis Prefix | Horizon Redis Prefix
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This prefix will be used when storing all Horizon data in Redis. You | This prefix will be used when storing all Horizon data in Redis. You
| may modify the prefix when you are running multiple installations | may modify the prefix when you are running multiple installations
| of Horizon on the same server so that they don't have problems. | of Horizon on the same server so that they don't have problems.
| |
*/ */
'prefix' => env('HORIZON_PREFIX', 'horizon-'.str_random(8).':'), 'prefix' => env('HORIZON_PREFIX', 'horizon-'.str_random(8).':'),
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Horizon Route Middleware | Horizon Route Middleware
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| These middleware will get attached onto each Horizon route, giving you | These middleware will get attached onto each Horizon route, giving you
| the chance to add your own middleware to this list or change any of | the chance to add your own middleware to this list or change any of
| the existing middleware. Or, you can simply stick with this list. | the existing middleware. Or, you can simply stick with this list.
| |
*/ */
'middleware' => ['web'], 'middleware' => ['web'],
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Queue Wait Time Thresholds | Queue Wait Time Thresholds
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This option allows you to configure when the LongWaitDetected event | This option allows you to configure when the LongWaitDetected event
| will be fired. Every connection / queue combination may have its | will be fired. Every connection / queue combination may have its
| own, unique threshold (in seconds) before this event is fired. | own, unique threshold (in seconds) before this event is fired.
| |
*/ */
'waits' => [ 'waits' => [
'redis:feed' => 30, 'redis:feed' => 30,
'redis:default' => 30, 'redis:default' => 30,
'redis:high' => 30, 'redis:high' => 30,
], 'redis:delete' => 30
],
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Job Trimming Times | Job Trimming Times
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| Here you can configure for how long (in minutes) you desire Horizon to | Here you can configure for how long (in minutes) you desire Horizon to
| persist the recent and failed jobs. Typically, recent jobs are kept | persist the recent and failed jobs. Typically, recent jobs are kept
| for one hour while all failed jobs are stored for an entire week. | for one hour while all failed jobs are stored for an entire week.
| |
*/ */
'trim' => [ 'trim' => [
'recent' => 60, 'recent' => 60,
'pending' => 60, 'pending' => 60,
'completed' => 60, 'completed' => 60,
'recent_failed' => 10080, 'recent_failed' => 10080,
'failed' => 10080, 'failed' => 10080,
'monitored' => 10080, 'monitored' => 10080,
], ],
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Metrics | Metrics
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| Here you can configure how many snapshots should be kept to display in | Here you can configure how many snapshots should be kept to display in
| the metrics graph. This will get used in combination with Horizon's | the metrics graph. This will get used in combination with Horizon's
| `horizon:snapshot` schedule to define how long to retain metrics. | `horizon:snapshot` schedule to define how long to retain metrics.
| |
*/ */
'metrics' => [ 'metrics' => [
'trim_snapshots' => [ 'trim_snapshots' => [
'job' => 24, 'job' => 24,
'queue' => 24, 'queue' => 24,
], ],
], ],
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Fast Termination | Fast Termination
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| When this option is enabled, Horizon's "terminate" command will not | When this option is enabled, Horizon's "terminate" command will not
| wait on all of the workers to terminate unless the --wait option | wait on all of the workers to terminate unless the --wait option
| is provided. Fast termination can shorten deployment delay by | is provided. Fast termination can shorten deployment delay by
| allowing a new instance of Horizon to start while the last | allowing a new instance of Horizon to start while the last
| instance will continue to terminate each of its workers. | instance will continue to terminate each of its workers.
| |
*/ */
'fast_termination' => false, 'fast_termination' => false,
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Memory Limit (MB) | Memory Limit (MB)
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| This value describes the maximum amount of memory the Horizon worker | This value describes the maximum amount of memory the Horizon worker
| may consume before it is terminated and restarted. You should set | may consume before it is terminated and restarted. You should set
| this value according to the resources available to your server. | this value according to the resources available to your server.
| |
*/ */
'memory_limit' => 64, 'memory_limit' => 64,
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Queue Worker Configuration | Queue Worker Configuration
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| Here you may define the queue worker settings used by your application | Here you may define the queue worker settings used by your application
| in all environments. These supervisors and settings handle all your | in all environments. These supervisors and settings handle all your
| queued jobs and will be provisioned by Horizon during deployment. | queued jobs and will be provisioned by Horizon during deployment.
| |
*/ */
'environments' => [ 'environments' => [
'production' => [ 'production' => [
'supervisor-1' => [ 'supervisor-1' => [
'connection' => 'redis', 'connection' => 'redis',
'queue' => ['high', 'default', 'feed'], 'queue' => ['high', 'default', 'feed', 'delete'],
'balance' => 'auto', 'balance' => 'auto',
'maxProcesses' => 20, 'maxProcesses' => 20,
'memory' => 128, 'memory' => 128,
'tries' => 3, 'tries' => 3,
'nice' => 0, 'nice' => 0,
], ],
], ],
'local' => [ 'local' => [
'supervisor-1' => [ 'supervisor-1' => [
'connection' => 'redis', 'connection' => 'redis',
'queue' => ['high', 'default', 'feed'], 'queue' => ['high', 'default', 'feed', 'delete'],
'balance' => 'auto', 'balance' => 'auto',
'maxProcesses' => 20, 'maxProcesses' => 20,
'memory' => 128, 'memory' => 128,
'tries' => 3, 'tries' => 3,
'nice' => 0, 'nice' => 0,
], ],
], ],
], ],
'darkmode' => env('HORIZON_DARKMODE', false), 'darkmode' => env('HORIZON_DARKMODE', false),
]; ];