Merge pull request #4278 from pixelfed/staging

Staging

.env.docker

@@ -101,7 +101,6 @@ NODEINFO=true
 WEBFINGER=true
 
 ## S3
-FILESYSTEM_DRIVER=local
 FILESYSTEM_CLOUD=s3
 PF_ENABLE_CLOUD=false
 #AWS_ACCESS_KEY_ID=

.env.example

@@ -68,7 +68,6 @@ MAIL_FROM_NAME="Pixelfed"
 
 ## S3 Configuration (Post-Installer)
 PF_ENABLE_CLOUD=false
-FILESYSTEM_DRIVER=local
 FILESYSTEM_CLOUD=s3
 #AWS_ACCESS_KEY_ID=
 #AWS_SECRET_ACCESS_KEY=

CHANGELOG.md

@@ -2,12 +2,19 @@
 
 ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.5...dev)
 
+### Added
+- New media:fix-nonlocal-driver command. Fixes s3 media created with invalid FILESYSTEM_DRIVER=s3 configuration ([672cccd4](https://github.com/pixelfed/pixelfed/commit/672cccd4))
+
 ### Updates
 - Update ApiV1Controller, fix blocking remote accounts. Closes #4256 ([8e71e0c0](https://github.com/pixelfed/pixelfed/commit/8e71e0c0))
 - Update ComposeController, fix postgres location search. Closes #4242 and #4239 ([64a4a006](https://github.com/pixelfed/pixelfed/commit/64a4a006))
 - Update app.js, add title attribute to iframe embeds to comply with accessibility requirements ([4d72b9e3](https://github.com/pixelfed/pixelfed/commit/4d72b9e3))
 - Update MediaPathService, fix story path ([aebbad96](https://github.com/pixelfed/pixelfed/commit/aebbad96))
 - Update Story v1.1 api endpoints ([855e9626](https://github.com/pixelfed/pixelfed/commit/855e9626))
+- Update ApiV1Controller, filter mute/blocks on statuses/context and statuses/replies endpoints ([73aa01e8](https://github.com/pixelfed/pixelfed/commit/73aa01e8))
+- Update filesystems, store all files as public by default and add default permissions. Fixes #4273, #4275. Closes #3825 ([22da2647](https://github.com/pixelfed/pixelfed/commit/22da2647))
+- Update Profile model, fix avatar url path generation. Fixes #4041, Fixes #4031, Fixes #3523 ([28bf8649](https://github.com/pixelfed/pixelfed/commit/28bf8649))
+- Update filesystem config, change FILESYSTEM_DRIVER env variable to DANGEROUSLY_SET_FILESYSTEM_DRIVER and remove from default env configs. Changing the default filesystem should be avoided, use FILESYSTEM_CLOUD for s3 support, otherwise you can break things ([573c88d7](https://github.com/pixelfed/pixelfed/commit/573c88d7))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.5 (2023-03-25)](https://github.com/pixelfed/pixelfed/compare/v0.11.4...v0.11.5)

app/Console/Commands/FixMediaDriver.php

@@ -0,0 +1,133 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Storage;
+use App\Media;
+use League\Flysystem\MountManager;
+use App\Jobs\ImageOptimizePipeline\ImageOptimize;
+use App\Jobs\MediaPipeline\MediaFixLocalFilesystemCleanupPipeline;
+
+class FixMediaDriver extends Command
+{
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'media:fix-nonlocal-driver';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Fix filesystem when FILESYSTEM_DRIVER not set to local';
+
+	/**
+	 * Execute the console command.
+	 *
+	 * @return int
+	 */
+	public function handle()
+	{
+		if(config('filesystems.default') !== 'local') {
+			$this->error('Invalid default filesystem, set FILESYSTEM_DRIVER=local to proceed');
+			return Command::SUCCESS;
+		}
+
+		if(config_cache('pixelfed.cloud_storage') == false) {
+			$this->error('Cloud storage not enabled, exiting...');
+			return Command::SUCCESS;
+		}
+
+		$this->info('       ____  _           ______         __  ');
+		$this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+		$this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+		$this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+		$this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+		$this->info(' ');
+		$this->info('   Media Filesystem Fix');
+		$this->info('   =====================');
+		$this->info('   Fix media that was created when FILESYSTEM_DRIVER=local');
+		$this->info('   was not properly set. This command will fix media urls');
+		$this->info('   and optionally optimize/generate thumbnails when applicable,');
+		$this->info('   clean up temporary local media files and clear the app cache');
+		$this->info('   to fix media paths/urls.');
+		$this->info(' ');
+		$this->error('   Remember, FILESYSTEM_DRIVER=local must remain set or you will break things!');
+
+		if(!$this->confirm('Are you sure you want to perform this command?')) {
+			$this->info('Exiting...');
+			return Command::SUCCESS;
+		}
+
+		$optimize = $this->choice(
+			'Do you want to optimize media and generate thumbnails? This will store s3 locally and re-upload optimized versions.',
+			['no', 'yes'],
+			1
+		);
+
+		$cloud = Storage::disk(config('filesystems.cloud'));
+		$mountManager = new MountManager([
+			's3' => $cloud->getDriver(),
+			'local' => Storage::disk('local')->getDriver(),
+		]);
+
+		$this->info('Fixing media, this may take a while...');
+		$this->line(' ');
+		$bar = $this->output->createProgressBar(Media::whereNotNull('status_id')->whereNull('cdn_url')->count());
+		$bar->start();
+
+		foreach(Media::whereNotNull('status_id')->whereNull('cdn_url')->lazyById(20) as $media) {
+			if($cloud->exists($media->media_path)) {
+				if($optimize === 'yes') {
+					$mountManager->copy(
+						's3://' . $media->media_path,
+						'local://' . $media->media_path
+					);
+					sleep(1);
+					if(empty($media->original_sha256)) {
+						$hash = \hash_file('sha256', Storage::disk('local')->path($media->media_path));
+						$media->original_sha256 = $hash;
+						$media->save();
+						sleep(1);
+					}
+					if(
+						$media->mime &&
+						in_array($media->mime, [
+							'image/jpeg',
+							'image/png',
+							'image/webp'
+						])
+					) {
+						ImageOptimize::dispatchSync($media);
+						sleep(3);
+					}
+				} else {
+					$media->cdn_url = $cloud->url($media->media_path);
+					$media->save();
+				}
+			}
+			$bar->advance();
+		}
+
+		$bar->finish();
+		$this->line(' ');
+		$this->line(' ');
+
+		$this->callSilently('cache:clear');
+
+		$this->info('Successfully fixed media paths and cleared cached!');
+
+		if($optimize === 'yes') {
+			MediaFixLocalFilesystemCleanupPipeline::dispatch()->delay(now()->addMinutes(15))->onQueue('default');
+			$this->line(' ');
+			$this->info('A cleanup job has been dispatched to delete media stored locally, it may take a few minutes to process!');
+		}
+
+		$this->line(' ');
+		return Command::SUCCESS;
+	}
+}

app/Jobs/MediaPipeline/MediaFixLocalFilesystemCleanupPipeline.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace App\Jobs\MediaPipeline;
+
+use App\Media;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Redis;
+use Illuminate\Support\Facades\Storage;
+
+class MediaFixLocalFilesystemCleanupPipeline implements ShouldQueue
+{
+	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+	public $timeout = 1800;
+	public $tries = 5;
+	public $maxExceptions = 1;
+
+	public function handle()
+	{
+		if(config_cache('pixelfed.cloud_storage') == false) {
+			// Only run if cloud storage is enabled
+			return;
+		}
+
+		$disk = Storage::disk('local');
+		$cloud = Storage::disk(config('filesystems.cloud'));
+
+		Media::whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
+		->chunk(20, function ($medias) use($disk, $cloud) {
+			foreach($medias as $media) {
+				if(!str_starts_with($media->media_path, 'public')) {
+					continue;
+				}
+
+				if($disk->exists($media->media_path) && $cloud->exists($media->media_path)) {
+					$disk->delete($media->media_path);
+				}
+
+				if($media->thumbnail_path) {
+					if($disk->exists($media->thumbnail_path)) {
+						$disk->delete($media->thumbnail_path);
+					}
+				}
+
+				$paths = explode('/', $media->media_path);
+				if(count($paths) === 7) {
+					array_pop($paths);
+					$baseDir = implode('/', $paths);
+
+					if(count($disk->allFiles($baseDir)) === 0) {
+						$disk->deleteDirectory($baseDir);
+
+						array_pop($paths);
+						$baseDir = implode('/', $paths);
+
+						if(count($disk->allFiles($baseDir)) === 0) {
+							$disk->deleteDirectory($baseDir);
+
+							array_pop($paths);
+							$baseDir = implode('/', $paths);
+
+							if(count($disk->allFiles($baseDir)) === 0) {
+								$disk->deleteDirectory($baseDir);
+							}
+						}
+					}
+				}
+			}
+		});
+	}
+}

app/Profile.php

@@ -178,13 +178,21 @@ class Profile extends Model
 				return url('/storage/avatars/default.jpg');
 			}
 
+			if($path === 'public/avatars/default.jpg') {
+				return url('/storage/avatars/default.jpg');
+			}
+
 			if(substr($path, 0, 6) !== 'public') {
 				return url('/storage/avatars/default.jpg');
 			}
 
+			if(config('filesystems.default') !== 'local') {
+				return Storage::url($path);
+			}
+
 			$path = "{$path}?v={$avatar->change_count}";
 
-			return config('app.url') . Storage::url($path);
+			return url(Storage::url($path));
 		});
 
 		return $url;

config/filesystems.php

@@ -13,7 +13,7 @@ return [
     |
     */
 
-    'default' => env('FILESYSTEM_DRIVER', 'local'),
+    'default' => env('DANGEROUSLY_SET_FILESYSTEM_DRIVER', 'local'),
 
     /*
     |--------------------------------------------------------------------------