Merge pull request #5440 from pixelfed/staging

Update Data Export, refactor following/follower and statuses exports to allow accounts of any size

CHANGELOG.md

@@ -33,6 +33,7 @@
 - Update BearerTokenResponse, return scopes in /oauth/token endpoint. Fixes #5286 ([d8f5c302](https://github.com/pixelfed/pixelfed/commit/d8f5c302))
 - Update hashtag component, fix missing video thumbnails ([witten](https://github.com/witten)) ([#5427](https://github.com/pixelfed/pixelfed/pull/5427))
 - Update AP Status Transformer, fix inReplyTo. Fixes #5409 ([83cc932f](https://github.com/pixelfed/pixelfed/commit/83cc932f))
+- Update Data Export, refactor following/follower and statuses exports to allow accounts of any size with api entity instead of ap ([0d25917c](https://github.com/pixelfed/pixelfed/commit/0d25917c))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.12.4 (2024-11-08)](https://github.com/pixelfed/pixelfed/compare/v0.12.4...dev)

app/Http/Controllers/Settings/ExportSettings.php

@@ -2,29 +2,27 @@
 
 namespace App\Http\Controllers\Settings;
 
-use App\AccountLog;
-use App\Following;
-use App\Report;
 use App\Status;
-use App\UserFilter;
-use Auth, Cookie, DB, Cache, Purify;
-use Carbon\Carbon;
-use Illuminate\Http\Request;
-use App\Transformer\ActivityPub\{
-	ProfileTransformer,
-	StatusTransformer
-};
+use App\Transformer\ActivityPub\ProfileTransformer;
 use App\Transformer\Api\StatusTransformer as StatusApiTransformer;
+use App\UserFilter;
+use Auth;
+use Cache;
+use Illuminate\Http\Request;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use League\Fractal\Pagination\IlluminatePaginatorAdapter;
+use Storage;
 
 trait ExportSettings
 {
-	public function __construct()
-	{
-		$this->middleware('auth');
-	}
+    private const CHUNK_SIZE = 1000;
+
+    private const STORAGE_BASE = 'user_exports';
+
+    public function __construct()
+    {
+        $this->middleware('auth');
+    }
 
     public function dataExport()
     {
@@ -33,47 +31,146 @@ trait ExportSettings
 
     public function exportAccount()
     {
-    	$data = Cache::remember('account:export:profile:actor:'.Auth::user()->profile->id, now()->addMinutes(60), function() {
-			$profile = Auth::user()->profile;
-			$fractal = new Fractal\Manager();
-			$fractal->setSerializer(new ArraySerializer());
-			$resource = new Fractal\Resource\Item($profile, new ProfileTransformer());
-			return $fractal->createData($resource)->toArray();
-    	});
+        $profile = Auth::user()->profile;
+        $fractal = new Fractal\Manager;
+        $fractal->setSerializer(new ArraySerializer);
+        $resource = new Fractal\Resource\Item($profile, new ProfileTransformer);
 
-    	return response()->streamDownload(function () use ($data) {
-    		echo json_encode($data, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE);
-    	}, 'account.json', [
-    		'Content-Type' => 'application/json'
-    	]);
+        $data = $fractal->createData($resource)->toArray();
+
+        return response()->streamDownload(function () use ($data) {
+            echo json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+        }, 'account.json', [
+            'Content-Type' => 'application/json',
+        ]);
     }
 
     public function exportFollowing()
     {
-        $data = Cache::remember('account:export:profile:following:'.Auth::user()->profile->id, now()->addMinutes(60), function() {
-            return Auth::user()->profile->following()->get()->map(function($i) {
-                return $i->url();
-            });
-        });
-        return response()->streamDownload(function () use($data) {
-            echo $data;
-        }, 'following.json', [
-    		'Content-Type' => 'application/json'
-    	]);
+        $profile = Auth::user()->profile;
+        $userId = Auth::id();
+
+        $userExportPath = 'user_exports/'.$userId;
+        $filename = 'pixelfed-following.json';
+        $tempPath = $userExportPath.'/'.$filename;
+
+        if (! Storage::exists($userExportPath)) {
+            Storage::makeDirectory($userExportPath);
+        }
+
+        try {
+            Storage::put($tempPath, '[');
+
+            $profile->following()
+                ->chunk(1000, function ($following) use ($tempPath) {
+                    $urls = $following->map(function ($follow) {
+                        return $follow->url();
+                    });
+
+                    $json = json_encode($urls,
+                        JSON_PRETTY_PRINT |
+                        JSON_UNESCAPED_SLASHES |
+                        JSON_UNESCAPED_UNICODE
+                    );
+
+                    $json = trim($json, '[]');
+                    if (Storage::size($tempPath) > 1) {
+                        $json = ','.$json;
+                    }
+
+                    Storage::append($tempPath, $json);
+                });
+
+            Storage::append($tempPath, ']');
+
+            return response()->stream(
+                function () use ($tempPath) {
+                    $handle = fopen(Storage::path($tempPath), 'rb');
+                    while (! feof($handle)) {
+                        echo fread($handle, 8192);
+                        flush();
+                    }
+                    fclose($handle);
+
+                    Storage::delete($tempPath);
+                },
+                200,
+                [
+                    'Content-Type' => 'application/json',
+                    'Content-Disposition' => 'attachment; filename="pixelfed-following.json"',
+                ]
+            );
+
+        } catch (\Exception $e) {
+            if (Storage::exists($tempPath)) {
+                Storage::delete($tempPath);
+            }
+            throw $e;
+        }
     }
 
     public function exportFollowers()
     {
-        $data = Cache::remember('account:export:profile:followers:'.Auth::user()->profile->id, now()->addMinutes(60), function() {
-            return Auth::user()->profile->followers()->get()->map(function($i) {
-                return $i->url();
-            });
-        });
-        return response()->streamDownload(function () use($data) {
-            echo $data;
-        }, 'followers.json', [
-    		'Content-Type' => 'application/json'
-    	]);
+        $profile = Auth::user()->profile;
+        $userId = Auth::id();
+
+        $userExportPath = 'user_exports/'.$userId;
+        $filename = 'pixelfed-followers.json';
+        $tempPath = $userExportPath.'/'.$filename;
+
+        if (! Storage::exists($userExportPath)) {
+            Storage::makeDirectory($userExportPath);
+        }
+
+        try {
+            Storage::put($tempPath, '[');
+
+            $profile->followers()
+                ->chunk(1000, function ($followers) use ($tempPath) {
+                    $urls = $followers->map(function ($follower) {
+                        return $follower->url();
+                    });
+
+                    $json = json_encode($urls,
+                        JSON_PRETTY_PRINT |
+                        JSON_UNESCAPED_SLASHES |
+                        JSON_UNESCAPED_UNICODE
+                    );
+
+                    $json = trim($json, '[]');
+                    if (Storage::size($tempPath) > 1) {
+                        $json = ','.$json;
+                    }
+
+                    Storage::append($tempPath, $json);
+                });
+
+            Storage::append($tempPath, ']');
+
+            return response()->stream(
+                function () use ($tempPath) {
+                    $handle = fopen(Storage::path($tempPath), 'rb');
+                    while (! feof($handle)) {
+                        echo fread($handle, 8192);
+                        flush();
+                    }
+                    fclose($handle);
+
+                    Storage::delete($tempPath);
+                },
+                200,
+                [
+                    'Content-Type' => 'application/json',
+                    'Content-Disposition' => 'attachment; filename="pixelfed-followers.json"',
+                ]
+            );
+
+        } catch (\Exception $e) {
+            if (Storage::exists($tempPath)) {
+                Storage::delete($tempPath);
+            }
+            throw $e;
+        }
     }
 
     public function exportMuteBlockList()
@@ -82,63 +179,83 @@ trait ExportSettings
         $exists = UserFilter::select('id')
             ->whereUserId($profile->id)
             ->exists();
-        if(!$exists) {
+        if (! $exists) {
             return redirect()->back();
         }
-        $data = Cache::remember('account:export:profile:muteblocklist:'.Auth::user()->profile->id, now()->addMinutes(60), function() use($profile) {
+        $data = Cache::remember('account:export:profile:muteblocklist:'.Auth::user()->profile->id, now()->addMinutes(60), function () use ($profile) {
             return json_encode([
                 'muted' => $profile->mutedProfileUrls(),
-                'blocked' => $profile->blockedProfileUrls()
+                'blocked' => $profile->blockedProfileUrls(),
             ], JSON_PRETTY_PRINT);
         });
-        return response()->streamDownload(function () use($data) {
+
+        return response()->streamDownload(function () use ($data) {
             echo $data;
         }, 'muted-and-blocked-accounts.json', [
-    		'Content-Type' => 'application/json'
-    	]);
+            'Content-Type' => 'application/json',
+        ]);
     }
 
     public function exportStatuses(Request $request)
     {
-    	$this->validate($request, [
-    		'type' => 'required|string|in:ap,api'
-    	]);
-    	$limit = 500;
+        $profile = Auth::user()->profile;
+        $userId = Auth::id();
+        $userExportPath = self::STORAGE_BASE.'/'.$userId;
+        $filename = 'pixelfed-statuses.json';
+        $tempPath = $userExportPath.'/'.$filename;
 
-    	$profile = Auth::user()->profile;
-    	$type = 'ap';
+        if (! Storage::exists($userExportPath)) {
+            Storage::makeDirectory($userExportPath);
+        }
 
-    	$count = Status::select('id')->whereProfileId($profile->id)->count();
-    	if($count > $limit) {
-    		// fire background job
-    		return redirect('/settings/data-export')->with(['status' => 'You have more than '.$limit.' statuses, we do not support full account export yet.']);
-    	}
+        Storage::put($tempPath, '[');
+        $fractal = new Fractal\Manager;
+        $fractal->setSerializer(new ArraySerializer);
 
-    	$filename = 'outbox.json';
-		if($type == 'ap') {
-			$data = Cache::remember('account:export:profile:statuses:ap:'.Auth::user()->profile->id, now()->addHours(1), function() {
-				$profile = Auth::user()->profile->statuses;
-				$fractal = new Fractal\Manager();
-				$fractal->setSerializer(new ArraySerializer());
-				$resource = new Fractal\Resource\Collection($profile, new StatusTransformer());
-				return $fractal->createData($resource)->toArray();
-			});
-		} else {
-			$filename = 'api-statuses.json';
-			$data = Cache::remember('account:export:profile:statuses:api:'.Auth::user()->profile->id, now()->addHours(1), function() {
-				$profile = Auth::user()->profile->statuses;
-				$fractal = new Fractal\Manager();
-				$fractal->setSerializer(new ArraySerializer());
-				$resource = new Fractal\Resource\Collection($profile, new StatusApiTransformer());
-				return $fractal->createData($resource)->toArray();
-			});
-		}
+        try {
+            Status::whereProfileId($profile->id)
+                ->chunk(self::CHUNK_SIZE, function ($statuses) use ($fractal, $tempPath) {
+                    $resource = new Fractal\Resource\Collection($statuses, new StatusApiTransformer);
+                    $data = $fractal->createData($resource)->toArray();
 
-    	return response()->streamDownload(function () use ($data, $filename) {
-    		echo json_encode($data, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE);
-    	}, $filename, [
-    		'Content-Type' => 'application/json'
-    	]);
+                    $json = json_encode($data,
+                        JSON_PRETTY_PRINT |
+                        JSON_UNESCAPED_SLASHES |
+                        JSON_UNESCAPED_UNICODE
+                    );
+
+                    $json = trim($json, '[]');
+                    if (Storage::size($tempPath) > 1) {
+                        $json = ','.$json;
+                    }
+
+                    Storage::append($tempPath, $json);
+                });
+
+            Storage::append($tempPath, ']');
+
+            return response()->stream(
+                function () use ($tempPath) {
+                    $handle = fopen(Storage::path($tempPath), 'rb');
+                    while (! feof($handle)) {
+                        echo fread($handle, 8192);
+                        flush();
+                    }
+                    fclose($handle);
+                    Storage::delete($tempPath);
+                },
+                200,
+                [
+                    'Content-Type' => 'application/json',
+                    'Content-Disposition' => 'attachment; filename="pixelfed-statuses.json"',
+                ]
+            );
+
+        } catch (\Exception $e) {
+            if (Storage::exists($tempPath)) {
+                Storage::delete($tempPath);
+            }
+            throw $e;
+        }
     }
-
-}
+}

app/Transformer/Api/StatusTransformer.php

@@ -25,7 +25,7 @@ class StatusTransformer extends Fractal\TransformerAbstract
         $pid = request()->user()->profile_id;
         $taggedPeople = MediaTagService::get($status->id);
         $poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null;
-        $content = $status->caption ? nl2br(Autolink::create()->autolink($status->caption)) : "";
+        $content = $status->caption ? nl2br(Autolink::create()->autolink($status->caption)) : '';
 
         return [
             '_v' => 1,
@@ -33,8 +33,8 @@ class StatusTransformer extends Fractal\TransformerAbstract
             'shortcode' => HashidService::encode($status->id),
             'uri' => $status->url(),
             'url' => $status->url(),
-            'in_reply_to_id' => (string) $status->in_reply_to_id,
-            'in_reply_to_account_id' => (string) $status->in_reply_to_profile_id,
+            'in_reply_to_id' => $status->in_reply_to_id ? (string) $status->in_reply_to_id : null,
+            'in_reply_to_account_id' => $status->in_reply_to_profile_id ? (string) $status->in_reply_to_profile_id : null,
             'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id) : null,
             'content' => $content,
             'content_text' => $status->caption,