Merge pull request #2665 from pixelfed/staging

Update federation pipeline, add locks

CHANGELOG.md

@@ -45,6 +45,7 @@
 - Updated InboxPipeline, fail earlier for invalid public keys. Fixes ([#2648](https://github.com/pixelfed/pixelfed/issues/2648)). ([d1c5e9b8](https://github.com/pixelfed/pixelfed/commit/d1c5e9b8))
 - Updated Status model, refactor liked and shared methods to fix cache invalidation bug. ([f05c3b66](https://github.com/pixelfed/pixelfed/commit/f05c3b66))
 - Updated Timeline component, add inline reports modal. ([e64b4bd3](https://github.com/pixelfed/pixelfed/commit/e64b4bd3))
+- Updated federation pipeline, add locks. ([ddc76887](https://github.com/pixelfed/pixelfed/commit/ddc76887))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.10.10 (2021-01-28)](https://github.com/pixelfed/pixelfed/compare/v0.10.9...v0.10.10)

app/Jobs/InboxPipeline/InboxValidator.php

@@ -53,6 +53,15 @@ class InboxValidator implements ShouldQueue
 
         $profile = Profile::whereNull('domain')->whereUsername($username)->first();
 
+        if(isset($payload['id'])) {
+            $lockKey = hash('sha256', $payload['id']);
+            if(Cache::get($lockKey) !== null) {
+                // Job processed already
+                return 1;
+            }
+            Cache::put($lockKey, 1, 300);
+        }
+
         if(!isset($headers['signature']) || !isset($headers['date'])) {
             return;
         }

app/Jobs/InboxPipeline/InboxWorker.php

@@ -49,6 +49,15 @@ class InboxWorker implements ShouldQueue
         $headers = $this->headers;
         $payload = json_decode($this->payload, true, 8);
 
+        if(isset($payload['id'])) {
+            $lockKey = hash('sha256', $payload['id']);
+            if(Cache::get($lockKey) !== null) {
+                // Job processed already
+                return 1;
+            }
+            Cache::put($lockKey, 1, 300);
+        }
+
         if(!isset($headers['signature']) || !isset($headers['date'])) {
             return;
         }

app/Util/ActivityPub/Helpers.php

@@ -346,7 +346,20 @@ class Helpers {
 			$reply_to = null;
 		}
 		$ts = is_array($res['published']) ? $res['published'][0] : $res['published'];
-		$status = DB::transaction(function() use($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) {
+
+		$statusLockKey = 'helpers:status-lock:' . hash('sha256', $res['id']);
+		$status = Cache::lock($statusLockKey)
+			->get(function () use(
+				$profile, 
+				$res, 
+				$url, 
+				$ts, 
+				$reply_to, 
+				$cw, 
+				$scope, 
+				$id
+		) {
+			return DB::transaction(function() use($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) {
 				$status = new Status;
 				$status->profile_id = $profile->id;
 				$status->url = isset($res['url']) ? $res['url'] : $url;
@@ -368,6 +381,7 @@ class Helpers {
 				}
 				return $status;
 			});
+		});
 
 		return $status;
 	}
@@ -458,7 +472,9 @@ class Helpers {
 
 			$profile = Profile::whereRemoteUrl($res['id'])->first();
 			if(!$profile) {
-				$profile = DB::transaction(function() use($domain, $webfinger, $res, $runJobs) {
+				$profileLockKey = 'helpers:profile-lock:' . hash('sha256', $res['id']);
+				$profile = Cache::lock($profileLockKey)->get(function () use($domain, $webfinger, $res, $runJobs) {
+					return DB::transaction(function() use($domain, $webfinger, $res, $runJobs) {
 						$profile = new Profile();
 						$profile->domain = strtolower($domain);
 						$profile->username = strtolower(Purify::clean($webfinger));
@@ -478,6 +494,7 @@ class Helpers {
 						}
 						return $profile;
 					});
+				});
 			} else {
 				// Update info after 24 hours
 				if($profile->last_fetched_at == null || 

routes/web.php

@@ -244,13 +244,13 @@ Route::domain(config('pixelfed.domain.app'))->middleware(['validemail', 'twofact
     Route::group(['prefix' => 'i'], function () {
         Route::redirect('/', '/');
         Route::get('compose', 'StatusController@compose')->name('compose');
-        Route::post('comment', 'CommentController@store')->middleware('throttle:maxCommentsPerHour,60')->middleware('throttle:maxCommentsPerDay,1440');
+        Route::post('comment', 'CommentController@store')->middleware('throttle:maxCommentsPerDay,1440');
         Route::post('delete', 'StatusController@delete');
         Route::post('mute', 'AccountController@mute');
         Route::post('unmute', 'AccountController@unmute');
         Route::post('block', 'AccountController@block');
         Route::post('unblock', 'AccountController@unblock');
-        Route::post('like', 'LikeController@store')->middleware('throttle:maxLikesPerHour,60')->middleware('throttle:maxLikesPerDay,1440');
+        Route::post('like', 'LikeController@store')->middleware('throttle:maxLikesPerDay,1440');
         Route::post('share', 'StatusController@storeShare')->middleware('throttle:maxSharesPerHour,60')->middleware('throttle:maxSharesPerDay,1440');
         Route::post('follow', 'FollowerController@store');
         Route::post('bookmark', 'BookmarkController@store');