diff --git a/app/Http/Controllers/Settings/ExportSettings.php b/app/Http/Controllers/Settings/ExportSettings.php index 91f8c9760..bbbc2e7f2 100644 --- a/app/Http/Controllers/Settings/ExportSettings.php +++ b/app/Http/Controllers/Settings/ExportSettings.php @@ -2,29 +2,27 @@ namespace App\Http\Controllers\Settings; -use App\AccountLog; -use App\Following; -use App\Report; use App\Status; -use App\UserFilter; -use Auth, Cookie, DB, Cache, Purify; -use Carbon\Carbon; -use Illuminate\Http\Request; -use App\Transformer\ActivityPub\{ - ProfileTransformer, - StatusTransformer -}; +use App\Transformer\ActivityPub\ProfileTransformer; use App\Transformer\Api\StatusTransformer as StatusApiTransformer; +use App\UserFilter; +use Auth; +use Cache; +use Illuminate\Http\Request; use League\Fractal; use League\Fractal\Serializer\ArraySerializer; -use League\Fractal\Pagination\IlluminatePaginatorAdapter; +use Storage; trait ExportSettings { - public function __construct() - { - $this->middleware('auth'); - } + private const CHUNK_SIZE = 1000; + + private const STORAGE_BASE = 'user_exports'; + + public function __construct() + { + $this->middleware('auth'); + } public function dataExport() { @@ -33,47 +31,146 @@ trait ExportSettings public function exportAccount() { - $data = Cache::remember('account:export:profile:actor:'.Auth::user()->profile->id, now()->addMinutes(60), function() { - $profile = Auth::user()->profile; - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Item($profile, new ProfileTransformer()); - return $fractal->createData($resource)->toArray(); - }); + $profile = Auth::user()->profile; + $fractal = new Fractal\Manager; + $fractal->setSerializer(new ArraySerializer); + $resource = new Fractal\Resource\Item($profile, new ProfileTransformer); - return response()->streamDownload(function () use ($data) { - echo json_encode($data, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE); - }, 'account.json', [ - 'Content-Type' => 'application/json' - ]); + $data = $fractal->createData($resource)->toArray(); + + return response()->streamDownload(function () use ($data) { + echo json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + }, 'account.json', [ + 'Content-Type' => 'application/json', + ]); } public function exportFollowing() { - $data = Cache::remember('account:export:profile:following:'.Auth::user()->profile->id, now()->addMinutes(60), function() { - return Auth::user()->profile->following()->get()->map(function($i) { - return $i->url(); - }); - }); - return response()->streamDownload(function () use($data) { - echo $data; - }, 'following.json', [ - 'Content-Type' => 'application/json' - ]); + $profile = Auth::user()->profile; + $userId = Auth::id(); + + $userExportPath = 'user_exports/'.$userId; + $filename = 'pixelfed-following.json'; + $tempPath = $userExportPath.'/'.$filename; + + if (! Storage::exists($userExportPath)) { + Storage::makeDirectory($userExportPath); + } + + try { + Storage::put($tempPath, '['); + + $profile->following() + ->chunk(1000, function ($following) use ($tempPath) { + $urls = $following->map(function ($follow) { + return $follow->url(); + }); + + $json = json_encode($urls, + JSON_PRETTY_PRINT | + JSON_UNESCAPED_SLASHES | + JSON_UNESCAPED_UNICODE + ); + + $json = trim($json, '[]'); + if (Storage::size($tempPath) > 1) { + $json = ','.$json; + } + + Storage::append($tempPath, $json); + }); + + Storage::append($tempPath, ']'); + + return response()->stream( + function () use ($tempPath) { + $handle = fopen(Storage::path($tempPath), 'rb'); + while (! feof($handle)) { + echo fread($handle, 8192); + flush(); + } + fclose($handle); + + Storage::delete($tempPath); + }, + 200, + [ + 'Content-Type' => 'application/json', + 'Content-Disposition' => 'attachment; filename="pixelfed-following.json"', + ] + ); + + } catch (\Exception $e) { + if (Storage::exists($tempPath)) { + Storage::delete($tempPath); + } + throw $e; + } } public function exportFollowers() { - $data = Cache::remember('account:export:profile:followers:'.Auth::user()->profile->id, now()->addMinutes(60), function() { - return Auth::user()->profile->followers()->get()->map(function($i) { - return $i->url(); - }); - }); - return response()->streamDownload(function () use($data) { - echo $data; - }, 'followers.json', [ - 'Content-Type' => 'application/json' - ]); + $profile = Auth::user()->profile; + $userId = Auth::id(); + + $userExportPath = 'user_exports/'.$userId; + $filename = 'pixelfed-followers.json'; + $tempPath = $userExportPath.'/'.$filename; + + if (! Storage::exists($userExportPath)) { + Storage::makeDirectory($userExportPath); + } + + try { + Storage::put($tempPath, '['); + + $profile->followers() + ->chunk(1000, function ($followers) use ($tempPath) { + $urls = $followers->map(function ($follower) { + return $follower->url(); + }); + + $json = json_encode($urls, + JSON_PRETTY_PRINT | + JSON_UNESCAPED_SLASHES | + JSON_UNESCAPED_UNICODE + ); + + $json = trim($json, '[]'); + if (Storage::size($tempPath) > 1) { + $json = ','.$json; + } + + Storage::append($tempPath, $json); + }); + + Storage::append($tempPath, ']'); + + return response()->stream( + function () use ($tempPath) { + $handle = fopen(Storage::path($tempPath), 'rb'); + while (! feof($handle)) { + echo fread($handle, 8192); + flush(); + } + fclose($handle); + + Storage::delete($tempPath); + }, + 200, + [ + 'Content-Type' => 'application/json', + 'Content-Disposition' => 'attachment; filename="pixelfed-followers.json"', + ] + ); + + } catch (\Exception $e) { + if (Storage::exists($tempPath)) { + Storage::delete($tempPath); + } + throw $e; + } } public function exportMuteBlockList() @@ -82,63 +179,83 @@ trait ExportSettings $exists = UserFilter::select('id') ->whereUserId($profile->id) ->exists(); - if(!$exists) { + if (! $exists) { return redirect()->back(); } - $data = Cache::remember('account:export:profile:muteblocklist:'.Auth::user()->profile->id, now()->addMinutes(60), function() use($profile) { + $data = Cache::remember('account:export:profile:muteblocklist:'.Auth::user()->profile->id, now()->addMinutes(60), function () use ($profile) { return json_encode([ 'muted' => $profile->mutedProfileUrls(), - 'blocked' => $profile->blockedProfileUrls() + 'blocked' => $profile->blockedProfileUrls(), ], JSON_PRETTY_PRINT); }); - return response()->streamDownload(function () use($data) { + + return response()->streamDownload(function () use ($data) { echo $data; }, 'muted-and-blocked-accounts.json', [ - 'Content-Type' => 'application/json' - ]); + 'Content-Type' => 'application/json', + ]); } public function exportStatuses(Request $request) { - $this->validate($request, [ - 'type' => 'required|string|in:ap,api' - ]); - $limit = 500; + $profile = Auth::user()->profile; + $userId = Auth::id(); + $userExportPath = self::STORAGE_BASE.'/'.$userId; + $filename = 'pixelfed-statuses.json'; + $tempPath = $userExportPath.'/'.$filename; - $profile = Auth::user()->profile; - $type = 'ap'; + if (! Storage::exists($userExportPath)) { + Storage::makeDirectory($userExportPath); + } - $count = Status::select('id')->whereProfileId($profile->id)->count(); - if($count > $limit) { - // fire background job - return redirect('/settings/data-export')->with(['status' => 'You have more than '.$limit.' statuses, we do not support full account export yet.']); - } + Storage::put($tempPath, '['); + $fractal = new Fractal\Manager; + $fractal->setSerializer(new ArraySerializer); - $filename = 'outbox.json'; - if($type == 'ap') { - $data = Cache::remember('account:export:profile:statuses:ap:'.Auth::user()->profile->id, now()->addHours(1), function() { - $profile = Auth::user()->profile->statuses; - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Collection($profile, new StatusTransformer()); - return $fractal->createData($resource)->toArray(); - }); - } else { - $filename = 'api-statuses.json'; - $data = Cache::remember('account:export:profile:statuses:api:'.Auth::user()->profile->id, now()->addHours(1), function() { - $profile = Auth::user()->profile->statuses; - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Collection($profile, new StatusApiTransformer()); - return $fractal->createData($resource)->toArray(); - }); - } + try { + Status::whereProfileId($profile->id) + ->chunk(self::CHUNK_SIZE, function ($statuses) use ($fractal, $tempPath) { + $resource = new Fractal\Resource\Collection($statuses, new StatusApiTransformer); + $data = $fractal->createData($resource)->toArray(); - return response()->streamDownload(function () use ($data, $filename) { - echo json_encode($data, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE); - }, $filename, [ - 'Content-Type' => 'application/json' - ]); + $json = json_encode($data, + JSON_PRETTY_PRINT | + JSON_UNESCAPED_SLASHES | + JSON_UNESCAPED_UNICODE + ); + + $json = trim($json, '[]'); + if (Storage::size($tempPath) > 1) { + $json = ','.$json; + } + + Storage::append($tempPath, $json); + }); + + Storage::append($tempPath, ']'); + + return response()->stream( + function () use ($tempPath) { + $handle = fopen(Storage::path($tempPath), 'rb'); + while (! feof($handle)) { + echo fread($handle, 8192); + flush(); + } + fclose($handle); + Storage::delete($tempPath); + }, + 200, + [ + 'Content-Type' => 'application/json', + 'Content-Disposition' => 'attachment; filename="pixelfed-statuses.json"', + ] + ); + + } catch (\Exception $e) { + if (Storage::exists($tempPath)) { + Storage::delete($tempPath); + } + throw $e; + } } - -} \ No newline at end of file +} diff --git a/app/Transformer/Api/StatusTransformer.php b/app/Transformer/Api/StatusTransformer.php index 4dc08b618..4a6dc5d7d 100644 --- a/app/Transformer/Api/StatusTransformer.php +++ b/app/Transformer/Api/StatusTransformer.php @@ -25,7 +25,7 @@ class StatusTransformer extends Fractal\TransformerAbstract $pid = request()->user()->profile_id; $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null; - $content = $status->caption ? nl2br(Autolink::create()->autolink($status->caption)) : ""; + $content = $status->caption ? nl2br(Autolink::create()->autolink($status->caption)) : ''; return [ '_v' => 1, @@ -33,8 +33,8 @@ class StatusTransformer extends Fractal\TransformerAbstract 'shortcode' => HashidService::encode($status->id), 'uri' => $status->url(), 'url' => $status->url(), - 'in_reply_to_id' => (string) $status->in_reply_to_id, - 'in_reply_to_account_id' => (string) $status->in_reply_to_profile_id, + 'in_reply_to_id' => $status->in_reply_to_id ? (string) $status->in_reply_to_id : null, + 'in_reply_to_account_id' => $status->in_reply_to_profile_id ? (string) $status->in_reply_to_profile_id : null, 'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id) : null, 'content' => $content, 'content_text' => $status->caption,