UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-22 14:31:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add Resilient Media Storage

Browse source
This commit is contained in:
Daniel Supernault 2023-09-25 00:59:24 -06:00
parent dcdfb28dcd
commit fb1deb6e28
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
4 changed files with 119 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
app/Services/MediaStorageService.php
View file

@ -86,12 +86,11 @@ class MediaStorageService {
$thumbname = array_pop($pt); $thumbname = array_pop($pt);
$storagePath = implode('/', $p); $storagePath = implode('/', $p);
$disk = Storage::disk(config('filesystems.cloud')); $url = ResilientMediaStorageService::store($storagePath, $path, $name);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); if($thumb) {
$url = $disk->url($file); $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public'); $media->thumbnail_url = $thumbUrl;
$thumbUrl = $disk->url($thumbFile); }
$media->thumbnail_url = $thumbUrl;
$media->cdn_url = $url; $media->cdn_url = $url;
$media->optimized_url = $url; $media->optimized_url = $url;
$media->replicated_at = now(); $media->replicated_at = now();

66
app/Services/ResilientMediaStorageService.php Normal file
View file

@ -0,0 +1,66 @@
<?php
namespace App\Services;
use Storage;
use Illuminate\Http\File;
use Exception;
use GuzzleHttp\Exception\ClientException;
use Aws\S3\Exception\S3Exception;
use GuzzleHttp\Exception\ConnectException;
use League\Flysystem\UnableToWriteFile;
class ResilientMediaStorageService
{
static $attempts = 0;
public static function store($storagePath, $path, $name)
{
return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
self::handleResilientStore($storagePath, $path, $name) :
self::handleStore($storagePath, $path, $name);
}
public static function handleStore($storagePath, $path, $name)
{
return retry(3, function() use($storagePath, $path, $name) {
$baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
return $disk->url($file);
}, random_int(100, 500));
}
public static function handleResilientStore($storagePath, $path, $name)
{
$attempts = 0;
return retry(4, function() use($storagePath, $path, $name, $attempts) {
self::$attempts++;
usleep(100000);
$baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
try {
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
} catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
return $disk->url($file);
}, function (int $attempt, Exception $exception) {
return $attempt * 200;
});
}
public static function getAltDriver()
{
$drivers = [];
if(config('filesystems.disks.alt-primary.enabled')) {
$drivers[] = 'alt-primary';
}
if(config('filesystems.disks.alt-secondary.enabled')) {
$drivers[] = 'alt-secondary';
}
if(empty($drivers)) {
return false;
}
$key = array_rand($drivers, 1);
return $drivers[$key];
}
}

28
config/filesystems.php
View file

@ -79,6 +79,34 @@ return [
'throw' => true, 'throw' => true,
], ],
'alt-primary' => [
'enabled' => env('ALT_PRI_ENABLED', false),
'driver' => 's3',
'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
'region' => env('ALT_PRI_AWS_DEFAULT_REGION'),
'bucket' => env('ALT_PRI_AWS_BUCKET'),
'visibility' => 'public',
'url' => env('ALT_PRI_AWS_URL'),
'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
'throw' => true,
],
'alt-secondary' => [
'enabled' => env('ALT_SEC_ENABLED', false),
'driver' => 's3',
'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
'region' => env('ALT_SEC_AWS_DEFAULT_REGION'),
'bucket' => env('ALT_SEC_AWS_BUCKET'),
'visibility' => 'public',
'url' => env('ALT_SEC_AWS_URL'),
'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
'throw' => true,
],
'spaces' => [ 'spaces' => [
'driver' => 's3', 'driver' => 's3',
'key' => env('DO_SPACES_KEY'), 'key' => env('DO_SPACES_KEY'),

38
config/media.php
View file

@ -1,24 +1,26 @@
<?php <?php
return [ return [
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), 'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
'exif' => [ 'exif' => [
'database' => env('MEDIA_EXIF_DATABASE', false), 'database' => env('MEDIA_EXIF_DATABASE', false),
], ],
'storage' => [ 'storage' => [
'remote' => [ 'remote' => [
/* /*
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| Store remote media on cloud/S3 | Store remote media on cloud/S3
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
| |
| Set this to cache remote media on cloud/S3 filesystem drivers. | Set this to cache remote media on cloud/S3 filesystem drivers.
| Disabled by default. | Disabled by default.
| |
*/ */
'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false) 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
],
] 'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
],
]
]; ];