Merge pull request #5245 from pixelfed/staging

Staging

app/Http/Controllers/Api/ApiV1Controller.php

@@ -60,6 +60,7 @@ use App\Services\SnowflakeService;
 use App\Services\StatusService;
 use App\Services\UserFilterService;
 use App\Services\UserRoleService;
+use App\Services\UserStorageService;
 use App\Status;
 use App\StatusHashtag;
 use App\Transformer\Api\Mastodon\v1\AccountTransformer;
@@ -1806,12 +1807,16 @@ class ApiV1Controller extends Controller
 
         $profile = $user->profile;
 
-        if (config_cache('pixelfed.enforce_account_limit') == true) {
+        $accountSize = UserStorageService::get($user->id);
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function () use ($user) {
+        abort_if($accountSize === -1, 403, 'Invalid request.');
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $photo = $request->file('file');
-            });
+        $fileSize = $photo->getSize();
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
@@ -1819,8 +1824,6 @@ class ApiV1Controller extends Controller
         $filterClass = in_array($request->input('filter_class'), Filter::classes()) ? $request->input('filter_class') : null;
         $filterName = in_array($request->input('filter_name'), Filter::names()) ? $request->input('filter_name') : null;
 
-        $photo = $request->file('file');
-
         $mimes = explode(',', config_cache('pixelfed.media_types'));
         if (in_array($photo->getMimeType(), $mimes) == false) {
             abort(403, 'Invalid or unsupported mime type.');
@@ -1883,6 +1886,10 @@ class ApiV1Controller extends Controller
                 break;
         }
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         Cache::forget($limitKey);
         $resource = new Fractal\Resource\Item($media, new MediaTransformer());
         $res = $this->fractal->createData($resource)->toArray();
@@ -2023,12 +2030,16 @@ class ApiV1Controller extends Controller
 
         $profile = $user->profile;
 
-        if (config_cache('pixelfed.enforce_account_limit') == true) {
+        $accountSize = UserStorageService::get($user->id);
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function () use ($user) {
+        abort_if($accountSize === -1, 403, 'Invalid request.');
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $photo = $request->file('file');
-            });
+        $fileSize = $photo->getSize();
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
@@ -2036,8 +2047,6 @@ class ApiV1Controller extends Controller
         $filterClass = in_array($request->input('filter_class'), Filter::classes()) ? $request->input('filter_class') : null;
         $filterName = in_array($request->input('filter_name'), Filter::names()) ? $request->input('filter_name') : null;
 
-        $photo = $request->file('file');
-
         $mimes = explode(',', config_cache('pixelfed.media_types'));
         if (in_array($photo->getMimeType(), $mimes) == false) {
             abort(403, 'Invalid or unsupported mime type.');
@@ -2105,6 +2114,10 @@ class ApiV1Controller extends Controller
                 break;
         }
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         Cache::forget($limitKey);
         $resource = new Fractal\Resource\Item($media, new MediaTransformer());
         $res = $this->fractal->createData($resource)->toArray();

app/Http/Controllers/Api/ApiV1Dot1Controller.php

@@ -29,6 +29,7 @@ use App\Services\NetworkTimelineService;
 use App\Services\ProfileStatusService;
 use App\Services\PublicTimelineService;
 use App\Services\StatusService;
+use App\Services\UserStorageService;
 use App\Status;
 use App\StatusArchived;
 use App\User;
@@ -1134,17 +1135,20 @@ class ApiV1Dot1Controller extends Controller
 
         $profile = $user->profile;
 
-        if (config_cache('pixelfed.enforce_account_limit') == true) {
+        $limitKey = 'compose:rate-limit:media-upload:'.$user->id;
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function () use ($user) {
+        $photo = $request->file('file');
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $fileSize = $photo->getSize();
-            });
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $accountSize = UserStorageService::get($user->id);
+        abort_if($accountSize === -1, 403, 'Invalid request.');
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
-        $limitKey = 'compose:rate-limit:media-upload:'.$user->id;
-        $photo = $request->file('file');
 
         $mimes = explode(',', config_cache('pixelfed.media_types'));
         if (in_array($photo->getMimeType(), $mimes) == false) {
@@ -1227,6 +1231,10 @@ class ApiV1Dot1Controller extends Controller
                 break;
         }
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         NewStatusPipeline::dispatch($status);
 
         Cache::forget('user:account:id:'.$user->id);

app/Http/Controllers/Api/ApiV2Controller.php

@@ -2,45 +2,32 @@
 
 namespace App\Http\Controllers\Api;
 
-use Illuminate\Http\Request;
 use App\Http\Controllers\Controller;
+use App\Jobs\ImageOptimizePipeline\ImageOptimize;
+use App\Jobs\MediaPipeline\MediaDeletePipeline;
+use App\Jobs\VideoPipeline\VideoThumbnail;
 use App\Media;
-use App\UserSetting;
-use App\User;
-use Illuminate\Support\Facades\Cache;
-use Illuminate\Support\Facades\Storage;
 use App\Services\AccountService;
-use App\Services\BouncerService;
 use App\Services\InstanceService;
 use App\Services\MediaBlocklistService;
 use App\Services\MediaPathService;
 use App\Services\SearchApiV2Service;
+use App\Services\UserRoleService;
+use App\Services\UserStorageService;
+use App\Transformer\Api\Mastodon\v1\MediaTransformer;
+use App\User;
+use App\UserSetting;
 use App\Util\Media\Filter;
-use App\Jobs\MediaPipeline\MediaDeletePipeline;
+use App\Util\Site\Nodeinfo;
-use App\Jobs\VideoPipeline\{
+use Illuminate\Http\Request;
-    VideoOptimize,
+use Illuminate\Support\Facades\Cache;
-    VideoPostProcess,
+use Illuminate\Support\Facades\Storage;
-    VideoThumbnail
-};
-use App\Jobs\ImageOptimizePipeline\ImageOptimize;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use League\Fractal\Pagination\IlluminatePaginatorAdapter;
-use App\Transformer\Api\Mastodon\v1\{
-    AccountTransformer,
-    MediaTransformer,
-    NotificationTransformer,
-    StatusTransformer,
-};
-use App\Transformer\Api\{
-    RelationshipTransformer,
-};
-use App\Util\Site\Nodeinfo;
-use App\Services\UserRoleService;
 
 class ApiV2Controller extends Controller
 {
-    const PF_API_ENTITY_KEY = "_pe";
+    const PF_API_ENTITY_KEY = '_pe';
 
     public function json($res, $code = 200, $headers = [])
     {
@@ -54,6 +41,7 @@ class ApiV2Controller extends Controller
                 return AccountService::getMastodon(config_cache('instance.admin.pid'), true);
             }
             $admin = User::whereIsAdmin(true)->first();
+
             return $admin && isset($admin->profile_id) ?
                 AccountService::getMastodon($admin->profile_id, true) :
                 null;
@@ -64,9 +52,10 @@ class ApiV2Controller extends Controller
                 collect(json_decode(config_cache('app.rules'), true))
                     ->map(function ($rule, $key) {
                         $id = $key + 1;
+
                         return [
                             'id' => "{$id}",
-                        'text' => $rule
+                            'text' => $rule,
                         ];
                     })
                     ->toArray() : [];
@@ -81,22 +70,22 @@ class ApiV2Controller extends Controller
                 'description' => config_cache('app.short_description'),
                 'usage' => [
                     'users' => [
-                        'active_month' => (int) Nodeinfo::activeUsersMonthly()
+                        'active_month' => (int) Nodeinfo::activeUsersMonthly(),
-                    ]
+                    ],
                 ],
                 'thumbnail' => [
                     'url' => config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg')),
                     'blurhash' => InstanceService::headerBlurhash(),
                     'versions' => [
                         '@1x' => config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg')),
-                        '@2x' => config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg'))
+                        '@2x' => config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg')),
-                    ]
+                    ],
                 ],
                 'languages' => [config('app.locale')],
                 'configuration' => [
                     'urls' => [
                         'streaming' => null,
-                        'status' => null
+                        'status' => null,
                     ],
                     'vapid' => [
                         'public_key' => config('webpush.vapid.public_key'),
@@ -107,7 +96,7 @@ class ApiV2Controller extends Controller
                     'statuses' => [
                         'max_characters' => (int) config_cache('pixelfed.max_caption_length'),
                         'max_media_attachments' => (int) config_cache('pixelfed.max_album_length'),
-                        'characters_reserved_per_url' => 23
+                        'characters_reserved_per_url' => 23,
                     ],
                     'media_attachments' => [
                         'supported_mime_types' => explode(',', config_cache('pixelfed.media_types')),
@@ -115,7 +104,7 @@ class ApiV2Controller extends Controller
                         'image_matrix_limit' => 3686400,
                         'video_size_limit' => config_cache('pixelfed.max_photo_size') * 1024,
                         'video_frame_rate_limit' => 240,
-                        'video_matrix_limit' => 3686400
+                        'video_matrix_limit' => 3686400,
                     ],
                     'polls' => [
                         'max_options' => 0,
@@ -135,14 +124,15 @@ class ApiV2Controller extends Controller
                 ],
                 'contact' => [
                     'email' => config('instance.email'),
-                    'account' => $contact
+                    'account' => $contact,
                 ],
-                'rules' => $rules
+                'rules' => $rules,
             ];
         });
 
         $res['registrations']['enabled'] = (bool) config_cache('pixelfed.open_registration');
         $res['registrations']['approval_required'] = (bool) config_cache('instance.curated_registration.enabled');
+
         return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
     }
 
@@ -167,18 +157,19 @@ class ApiV2Controller extends Controller
             'resolve' => 'nullable',
             'limit' => 'nullable|integer|max:40',
             'offset' => 'nullable|integer',
-            'following' => 'nullable'
+            'following' => 'nullable',
         ]);
 
         if ($request->user()->has_roles && ! UserRoleService::can('can-view-discover', $request->user()->id)) {
             return [
                 'accounts' => [],
                 'hashtags' => [],
-                'statuses' => []
+                'statuses' => [],
             ];
         }
 
         $mastodonMode = ! $request->has('_pe');
+
         return $this->json(SearchApiV2Service::query($request, $mastodonMode));
     }
 
@@ -194,7 +185,7 @@ class ApiV2Controller extends Controller
             'host' => config('broadcasting.connections.pusher.options.host'),
             'port' => config('broadcasting.connections.pusher.options.port'),
             'key' => config('broadcasting.connections.pusher.key'),
-            'cluster' => config('broadcasting.connections.pusher.options.cluster')
+            'cluster' => config('broadcasting.connections.pusher.options.cluster'),
         ] : [];
     }
 
@@ -223,7 +214,7 @@ class ApiV2Controller extends Controller
             'filter_name' => 'nullable|string|max:24',
             'filter_class' => 'nullable|alpha_dash|max:24',
             'description' => 'nullable|string|max:'.config_cache('pixelfed.max_altext_length'),
-          'replace_id' => 'sometimes'
+            'replace_id' => 'sometimes',
         ]);
 
         $user = $request->user();
@@ -247,12 +238,16 @@ class ApiV2Controller extends Controller
 
         $profile = $user->profile;
 
-        if(config_cache('pixelfed.enforce_account_limit') == true) {
+        $accountSize = UserStorageService::get($user->id);
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function() use($user) {
+        abort_if($accountSize === -1, 403, 'Invalid request.');
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $photo = $request->file('file');
-            });
+        $fileSize = $photo->getSize();
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
@@ -260,8 +255,6 @@ class ApiV2Controller extends Controller
         $filterClass = in_array($request->input('filter_class'), Filter::classes()) ? $request->input('filter_class') : null;
         $filterName = in_array($request->input('filter_name'), Filter::names()) ? $request->input('filter_name') : null;
 
-        $photo = $request->file('file');
-
         $mimes = explode(',', config_cache('pixelfed.media_types'));
         if (in_array($photo->getMimeType(), $mimes) == false) {
             abort(403, 'Invalid or unsupported mime type.');
@@ -328,6 +321,10 @@ class ApiV2Controller extends Controller
                 break;
         }
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         Cache::forget($limitKey);
         $fractal = new Fractal\Manager();
         $fractal->setSerializer(new ArraySerializer());
@@ -335,6 +332,7 @@ class ApiV2Controller extends Controller
         $res = $fractal->createData($resource)->toArray();
         $res['preview_url'] = $media->url().'?v='.time();
         $res['url'] = null;
+
         return $this->json($res, 202);
     }
 }

app/Http/Controllers/ComposeController.php

@@ -21,6 +21,7 @@ use App\Services\MediaStorageService;
 use App\Services\MediaTagService;
 use App\Services\SnowflakeService;
 use App\Services\UserRoleService;
+use App\Services\UserStorageService;
 use App\Status;
 use App\Transformer\Api\MediaTransformer;
 use App\UserFilter;
@@ -70,7 +71,7 @@ class ComposeController extends Controller
             'filter_class' => 'nullable|alpha_dash|max:24',
         ]);
 
-        $user = Auth::user();
+        $user = $request->user();
         $profile = $user->profile;
         abort_if($user->has_roles && ! UserRoleService::can('can-post', $user->id), 403, 'Invalid permissions for this action');
 
@@ -84,21 +85,22 @@ class ComposeController extends Controller
 
         abort_if($limitReached == true, 429);
 
-        if (config_cache('pixelfed.enforce_account_limit') == true) {
+        $filterClass = in_array($request->input('filter_class'), Filter::classes()) ? $request->input('filter_class') : null;
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function () use ($user) {
+        $filterName = in_array($request->input('filter_name'), Filter::names()) ? $request->input('filter_name') : null;
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $accountSize = UserStorageService::get($user->id);
-            });
+        abort_if($accountSize === -1, 403, 'Invalid request.');
+        $photo = $request->file('file');
+        $fileSize = $photo->getSize();
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
 
-        $filterClass = in_array($request->input('filter_class'), Filter::classes()) ? $request->input('filter_class') : null;
-        $filterName = in_array($request->input('filter_name'), Filter::names()) ? $request->input('filter_name') : null;
-
-        $photo = $request->file('file');
-
         $mimes = explode(',', config_cache('pixelfed.media_types'));
 
         abort_if(in_array($photo->getMimeType(), $mimes) == false, 400, 'Invalid media format');
@@ -143,6 +145,10 @@ class ComposeController extends Controller
                 break;
         }
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         Cache::forget($limitKey);
         $resource = new Fractal\Resource\Item($media, new MediaTransformer());
         $res = $this->fractal->createData($resource)->toArray();
@@ -198,6 +204,7 @@ class ComposeController extends Controller
         ];
         ImageOptimize::dispatch($media)->onQueue('mmo');
         Cache::forget($limitKey);
+        UserStorageService::recalculateUpdateStorageUsed($request->user()->id);
 
         return $res;
     }
@@ -218,6 +225,8 @@ class ComposeController extends Controller
 
         MediaStorageService::delete($media, true);
 
+        UserStorageService::recalculateUpdateStorageUsed($request->user()->id);
+
         return response()->json([
             'msg' => 'Successfully deleted',
             'code' => 200,
@@ -494,17 +503,17 @@ class ComposeController extends Controller
 
         $limitKey = 'compose:rate-limit:store:'.$user->id;
         $limitTtl = now()->addMinutes(15);
-        $limitReached = Cache::remember($limitKey, $limitTtl, function () use ($user) {
+        // $limitReached = Cache::remember($limitKey, $limitTtl, function () use ($user) {
-            $dailyLimit = Status::whereProfileId($user->profile_id)
+        //     $dailyLimit = Status::whereProfileId($user->profile_id)
-                ->whereNull('in_reply_to_id')
+        //         ->whereNull('in_reply_to_id')
-                ->whereNull('reblog_of_id')
+        //         ->whereNull('reblog_of_id')
-                ->where('created_at', '>', now()->subDays(1))
+        //         ->where('created_at', '>', now()->subDays(1))
-                ->count();
+        //         ->count();
 
-            return $dailyLimit >= 1000;
+        //     return $dailyLimit >= 1000;
-        });
+        // });
 
-        abort_if($limitReached == true, 429);
+        // abort_if($limitReached == true, 429);
 
         $license = in_array($request->input('license'), License::keys()) ? $request->input('license') : null;
 
@@ -626,7 +635,6 @@ class ComposeController extends Controller
         Cache::forget('_api:statuses:recent_9:'.$profile->id);
         Cache::forget('profile:status_count:'.$profile->id);
         Cache::forget('status:transformer:media:attachments:'.$status->id);
-        Cache::forget($user->storageUsedKey());
         Cache::forget('profile:embed:'.$status->profile_id);
         Cache::forget($limitKey);
 

app/Http/Controllers/DirectMessageController.php

@@ -17,11 +17,11 @@ use App\Services\MediaService;
 use App\Services\StatusService;
 use App\Services\UserFilterService;
 use App\Services\UserRoleService;
+use App\Services\UserStorageService;
 use App\Services\WebfingerService;
 use App\Status;
 use App\UserFilter;
 use App\Util\ActivityPub\Helpers;
-use Cache;
 use Illuminate\Http\Request;
 use Illuminate\Support\Str;
 
@@ -602,16 +602,19 @@ class DirectMessageController extends Controller
             $hidden = false;
         }
 
-        if (config_cache('pixelfed.enforce_account_limit') == true) {
+        $accountSize = UserStorageService::get($user->id);
-            $size = Cache::remember($user->storageUsedKey(), now()->addDays(3), function () use ($user) {
+        abort_if($accountSize === -1, 403, 'Invalid request.');
-                return Media::whereUserId($user->id)->sum('size') / 1000;
+        $photo = $request->file('file');
-            });
+        $fileSize = $photo->getSize();
+        $sizeInKbs = (int) ceil($fileSize / 1000);
+        $updatedAccountSize = (int) $accountSize + (int) $sizeInKbs;
+
+        if ((bool) config_cache('pixelfed.enforce_account_limit') == true) {
             $limit = (int) config_cache('pixelfed.max_account_size');
-            if ($size >= $limit) {
+            if ($updatedAccountSize >= $limit) {
                 abort(403, 'Account size limit reached.');
             }
         }
-        $photo = $request->file('file');
 
         $mimes = explode(',', config_cache('pixelfed.media_types'));
         if (in_array($photo->getMimeType(), $mimes) == false) {
@@ -667,6 +670,10 @@ class DirectMessageController extends Controller
             ]
         );
 
+        $user->storage_used = (int) $updatedAccountSize;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
         if ($recipient->domain) {
             $this->remoteDeliver($dm);
         }

app/Services/UserStorageService.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace App\Services;
+
+use App\Media;
+use App\User;
+
+class UserStorageService
+{
+    const CACHE_KEY = 'pf:services:user-storage:byId:';
+
+    public static function get($id)
+    {
+        $user = User::find($id);
+        if (! $user || $user->status) {
+            return -1;
+        }
+
+        if ($user->storage_used_updated_at) {
+            return (int) $user->storage_used;
+        }
+        $updatedVal = self::calculateStorageUsed($id);
+        $user->storage_used = $updatedVal;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
+        return $user->storage_used;
+    }
+
+    public static function calculateStorageUsed($id)
+    {
+        return (int) floor(Media::whereUserId($id)->sum('size') / 1000);
+    }
+
+    public static function recalculateUpdateStorageUsed($id)
+    {
+        $user = User::find($id);
+        if (! $user || $user->status) {
+            return;
+        }
+        $updatedVal = (int) floor(Media::whereUserId($id)->sum('size') / 1000);
+        $user->storage_used = $updatedVal;
+        $user->storage_used_updated_at = now();
+        $user->save();
+
+        return $updatedVal;
+    }
+}

database/migrations/2024_07_29_081002_add_storage_used_to_users_table.php

@@ -0,0 +1,30 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('users', function (Blueprint $table) {
+            $table->unsignedBigInteger('storage_used')->default(0);
+            $table->timestamp('storage_used_updated_at')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('users', function (Blueprint $table) {
+            $table->dropColumn('storage_used');
+            $table->dropColumn('storage_used_updated_at');
+        });
+    }
+};