Merge pull request #4278 from pixelfed/staging

Staging
This commit is contained in:
daniel 2023-04-04 03:56:35 -06:00 committed by GitHub
commit 21714eec34
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 225 additions and 4 deletions

View file

@ -101,7 +101,6 @@ NODEINFO=true
WEBFINGER=true WEBFINGER=true
## S3 ## S3
FILESYSTEM_DRIVER=local
FILESYSTEM_CLOUD=s3 FILESYSTEM_CLOUD=s3
PF_ENABLE_CLOUD=false PF_ENABLE_CLOUD=false
#AWS_ACCESS_KEY_ID= #AWS_ACCESS_KEY_ID=

View file

@ -68,7 +68,6 @@ MAIL_FROM_NAME="Pixelfed"
## S3 Configuration (Post-Installer) ## S3 Configuration (Post-Installer)
PF_ENABLE_CLOUD=false PF_ENABLE_CLOUD=false
FILESYSTEM_DRIVER=local
FILESYSTEM_CLOUD=s3 FILESYSTEM_CLOUD=s3
#AWS_ACCESS_KEY_ID= #AWS_ACCESS_KEY_ID=
#AWS_SECRET_ACCESS_KEY= #AWS_SECRET_ACCESS_KEY=

View file

@ -2,12 +2,19 @@
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.5...dev) ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.5...dev)
### Added
- New media:fix-nonlocal-driver command. Fixes s3 media created with invalid FILESYSTEM_DRIVER=s3 configuration ([672cccd4](https://github.com/pixelfed/pixelfed/commit/672cccd4))
### Updates ### Updates
- Update ApiV1Controller, fix blocking remote accounts. Closes #4256 ([8e71e0c0](https://github.com/pixelfed/pixelfed/commit/8e71e0c0)) - Update ApiV1Controller, fix blocking remote accounts. Closes #4256 ([8e71e0c0](https://github.com/pixelfed/pixelfed/commit/8e71e0c0))
- Update ComposeController, fix postgres location search. Closes #4242 and #4239 ([64a4a006](https://github.com/pixelfed/pixelfed/commit/64a4a006)) - Update ComposeController, fix postgres location search. Closes #4242 and #4239 ([64a4a006](https://github.com/pixelfed/pixelfed/commit/64a4a006))
- Update app.js, add title attribute to iframe embeds to comply with accessibility requirements ([4d72b9e3](https://github.com/pixelfed/pixelfed/commit/4d72b9e3)) - Update app.js, add title attribute to iframe embeds to comply with accessibility requirements ([4d72b9e3](https://github.com/pixelfed/pixelfed/commit/4d72b9e3))
- Update MediaPathService, fix story path ([aebbad96](https://github.com/pixelfed/pixelfed/commit/aebbad96)) - Update MediaPathService, fix story path ([aebbad96](https://github.com/pixelfed/pixelfed/commit/aebbad96))
- Update Story v1.1 api endpoints ([855e9626](https://github.com/pixelfed/pixelfed/commit/855e9626)) - Update Story v1.1 api endpoints ([855e9626](https://github.com/pixelfed/pixelfed/commit/855e9626))
- Update ApiV1Controller, filter mute/blocks on statuses/context and statuses/replies endpoints ([73aa01e8](https://github.com/pixelfed/pixelfed/commit/73aa01e8))
- Update filesystems, store all files as public by default and add default permissions. Fixes #4273, #4275. Closes #3825 ([22da2647](https://github.com/pixelfed/pixelfed/commit/22da2647))
- Update Profile model, fix avatar url path generation. Fixes #4041, Fixes #4031, Fixes #3523 ([28bf8649](https://github.com/pixelfed/pixelfed/commit/28bf8649))
- Update filesystem config, change FILESYSTEM_DRIVER env variable to DANGEROUSLY_SET_FILESYSTEM_DRIVER and remove from default env configs. Changing the default filesystem should be avoided, use FILESYSTEM_CLOUD for s3 support, otherwise you can break things ([573c88d7](https://github.com/pixelfed/pixelfed/commit/573c88d7))
- ([](https://github.com/pixelfed/pixelfed/commit/)) - ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.5 (2023-03-25)](https://github.com/pixelfed/pixelfed/compare/v0.11.4...v0.11.5) ## [v0.11.5 (2023-03-25)](https://github.com/pixelfed/pixelfed/compare/v0.11.4...v0.11.5)

View file

@ -0,0 +1,133 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Storage;
use App\Media;
use League\Flysystem\MountManager;
use App\Jobs\ImageOptimizePipeline\ImageOptimize;
use App\Jobs\MediaPipeline\MediaFixLocalFilesystemCleanupPipeline;
class FixMediaDriver extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:fix-nonlocal-driver';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Fix filesystem when FILESYSTEM_DRIVER not set to local';
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
if(config('filesystems.default') !== 'local') {
$this->error('Invalid default filesystem, set FILESYSTEM_DRIVER=local to proceed');
return Command::SUCCESS;
}
if(config_cache('pixelfed.cloud_storage') == false) {
$this->error('Cloud storage not enabled, exiting...');
return Command::SUCCESS;
}
$this->info(' ____ _ ______ __ ');
$this->info(' / __ \(_) _____ / / __/__ ____/ / ');
$this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / ');
$this->info(' / ____/ /> </ __/ / __/ __/ /_/ / ');
$this->info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ ');
$this->info(' ');
$this->info(' Media Filesystem Fix');
$this->info(' =====================');
$this->info(' Fix media that was created when FILESYSTEM_DRIVER=local');
$this->info(' was not properly set. This command will fix media urls');
$this->info(' and optionally optimize/generate thumbnails when applicable,');
$this->info(' clean up temporary local media files and clear the app cache');
$this->info(' to fix media paths/urls.');
$this->info(' ');
$this->error(' Remember, FILESYSTEM_DRIVER=local must remain set or you will break things!');
if(!$this->confirm('Are you sure you want to perform this command?')) {
$this->info('Exiting...');
return Command::SUCCESS;
}
$optimize = $this->choice(
'Do you want to optimize media and generate thumbnails? This will store s3 locally and re-upload optimized versions.',
['no', 'yes'],
1
);
$cloud = Storage::disk(config('filesystems.cloud'));
$mountManager = new MountManager([
's3' => $cloud->getDriver(),
'local' => Storage::disk('local')->getDriver(),
]);
$this->info('Fixing media, this may take a while...');
$this->line(' ');
$bar = $this->output->createProgressBar(Media::whereNotNull('status_id')->whereNull('cdn_url')->count());
$bar->start();
foreach(Media::whereNotNull('status_id')->whereNull('cdn_url')->lazyById(20) as $media) {
if($cloud->exists($media->media_path)) {
if($optimize === 'yes') {
$mountManager->copy(
's3://' . $media->media_path,
'local://' . $media->media_path
);
sleep(1);
if(empty($media->original_sha256)) {
$hash = \hash_file('sha256', Storage::disk('local')->path($media->media_path));
$media->original_sha256 = $hash;
$media->save();
sleep(1);
}
if(
$media->mime &&
in_array($media->mime, [
'image/jpeg',
'image/png',
'image/webp'
])
) {
ImageOptimize::dispatchSync($media);
sleep(3);
}
} else {
$media->cdn_url = $cloud->url($media->media_path);
$media->save();
}
}
$bar->advance();
}
$bar->finish();
$this->line(' ');
$this->line(' ');
$this->callSilently('cache:clear');
$this->info('Successfully fixed media paths and cleared cached!');
if($optimize === 'yes') {
MediaFixLocalFilesystemCleanupPipeline::dispatch()->delay(now()->addMinutes(15))->onQueue('default');
$this->line(' ');
$this->info('A cleanup job has been dispatched to delete media stored locally, it may take a few minutes to process!');
}
$this->line(' ');
return Command::SUCCESS;
}
}

View file

@ -0,0 +1,75 @@
<?php
namespace App\Jobs\MediaPipeline;
use App\Media;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Storage;
class MediaFixLocalFilesystemCleanupPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 1800;
public $tries = 5;
public $maxExceptions = 1;
public function handle()
{
if(config_cache('pixelfed.cloud_storage') == false) {
// Only run if cloud storage is enabled
return;
}
$disk = Storage::disk('local');
$cloud = Storage::disk(config('filesystems.cloud'));
Media::whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->chunk(20, function ($medias) use($disk, $cloud) {
foreach($medias as $media) {
if(!str_starts_with($media->media_path, 'public')) {
continue;
}
if($disk->exists($media->media_path) && $cloud->exists($media->media_path)) {
$disk->delete($media->media_path);
}
if($media->thumbnail_path) {
if($disk->exists($media->thumbnail_path)) {
$disk->delete($media->thumbnail_path);
}
}
$paths = explode('/', $media->media_path);
if(count($paths) === 7) {
array_pop($paths);
$baseDir = implode('/', $paths);
if(count($disk->allFiles($baseDir)) === 0) {
$disk->deleteDirectory($baseDir);
array_pop($paths);
$baseDir = implode('/', $paths);
if(count($disk->allFiles($baseDir)) === 0) {
$disk->deleteDirectory($baseDir);
array_pop($paths);
$baseDir = implode('/', $paths);
if(count($disk->allFiles($baseDir)) === 0) {
$disk->deleteDirectory($baseDir);
}
}
}
}
}
});
}
}

View file

@ -178,13 +178,21 @@ class Profile extends Model
return url('/storage/avatars/default.jpg'); return url('/storage/avatars/default.jpg');
} }
if($path === 'public/avatars/default.jpg') {
return url('/storage/avatars/default.jpg');
}
if(substr($path, 0, 6) !== 'public') { if(substr($path, 0, 6) !== 'public') {
return url('/storage/avatars/default.jpg'); return url('/storage/avatars/default.jpg');
} }
if(config('filesystems.default') !== 'local') {
return Storage::url($path);
}
$path = "{$path}?v={$avatar->change_count}"; $path = "{$path}?v={$avatar->change_count}";
return config('app.url') . Storage::url($path); return url(Storage::url($path));
}); });
return $url; return $url;

View file

@ -13,7 +13,7 @@ return [
| |
*/ */
'default' => env('FILESYSTEM_DRIVER', 'local'), 'default' => env('DANGEROUSLY_SET_FILESYSTEM_DRIVER', 'local'),
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------