Merge pull request #4466 from pixelfed/staging

Import from Instagram
This commit is contained in:
daniel 2023-06-12 05:56:27 -06:00 committed by GitHub
commit d76ae33eb9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 2634 additions and 8011 deletions

View file

@ -2,6 +2,9 @@
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.8...dev)
### Added
- Import from Instagram ([#4466](https://github.com/pixelfed/pixelfed/pull/4466)) ([cf3078c5](https://github.com/pixelfed/pixelfed/commit/cf3078c5))
### Updates
- Update Notifications.vue component, fix filtering logic to prevent endless spinner ([3df9b53f](https://github.com/pixelfed/pixelfed/commit/3df9b53f))
- Update Direct Messages, fix api endpoint ([fe8728c0](https://github.com/pixelfed/pixelfed/commit/fe8728c0))

View file

@ -0,0 +1,49 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Models\ImportPost;
use Storage;
use App\Services\ImportService;
class ImportUploadGarbageCollection extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:import-upload-garbage-collection';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Execute the console command.
*/
public function handle()
{
if(!config('import.instagram.enabled')) {
return;
}
$ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(true)->take(100)->get();
if(!$ips->count()) {
return;
}
foreach($ips as $ip) {
$pid = $ip->profile_id;
$ip->delete();
ImportService::getPostCount($pid, true);
ImportService::clearAttempts($pid);
ImportService::getImportedFiles($pid, true);
}
}
}

View file

@ -0,0 +1,122 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Models\ImportPost;
use App\Services\ImportService;
use App\Media;
use App\Profile;
use App\Status;
use Storage;
use App\Services\MediaPathService;
use Illuminate\Support\Str;
use App\Util\Lexer\Autolink;
class TransformImports extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:transform-imports';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Transform imports into statuses';
/**
* Execute the console command.
*/
public function handle()
{
if(!config('import.instagram.enabled')) {
return;
}
$ips = ImportPost::whereNull('status_id')->whereSkipMissingMedia(false)->take(100)->get();
if(!$ips->count()) {
return;
}
foreach($ips as $ip) {
$id = $ip->user_id;
$pid = $ip->profile_id;
$profile = Profile::find($pid);
$idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day);
if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) {
ImportService::clearAttempts($profile->id);
ImportService::getPostCount($profile->id, true);
$ip->skip_missing_media = true;
$ip->save();
continue;
}
$missingMedia = false;
foreach($ip->media as $ipm) {
$fileName = last(explode('/', $ipm['uri']));
$og = 'imports/' . $id . '/' . $fileName;
if(!Storage::exists($og)) {
$missingMedia = true;
}
}
if($missingMedia === true) {
$ip->skip_missing_media = true;
$ip->save();
continue;
}
$caption = $ip->caption;
$status = new Status;
$status->profile_id = $pid;
$status->caption = $caption;
$status->rendered = strlen(trim($caption)) ? Autolink::create()->autolink($ip->caption) : null;
$status->type = $ip->post_type;
$status->scope = 'unlisted';
$status->visibility = 'unlisted';
$status->id = $idk['id'];
$status->created_at = now()->parse($ip->creation_date);
$status->save();
foreach($ip->media as $ipm) {
$fileName = last(explode('/', $ipm['uri']));
$ext = last(explode('.', $fileName));
$basePath = MediaPathService::get($profile);
$og = 'imports/' . $id . '/' . $fileName;
if(!Storage::exists($og)) {
$ip->skip_missing_media = true;
$ip->save();
continue;
}
$size = Storage::size($og);
$mime = Storage::mimeType($og);
$newFile = Str::random(40) . '.' . $ext;
$np = $basePath . '/' . $newFile;
Storage::move($og, $np);
$media = new Media;
$media->profile_id = $pid;
$media->user_id = $id;
$media->status_id = $status->id;
$media->media_path = $np;
$media->mime = $mime;
$media->size = $size;
$media->save();
}
$ip->status_id = $status->id;
$ip->creation_id = $idk['incr'];
$ip->save();
ImportService::clearAttempts($profile->id);
ImportService::getPostCount($profile->id, true);
}
}
}

View file

@ -36,6 +36,11 @@ class Kernel extends ConsoleKernel
if(in_array(config_cache('pixelfed.cloud_storage'), ['1', true, 'true']) && config('media.delete_local_after_cloud')) {
$schedule->command('media:s3gc')->hourlyAt(15);
}
if(config('import.instagram.enabled')) {
$schedule->command('app:transform-imports')->everyFourMinutes();
$schedule->command('app:import-upload-garbage-collection')->everyFiveMinutes();
}
}
/**

View file

@ -0,0 +1,298 @@
<?php
namespace App\Http\Controllers;
use Illuminate\Http\Request;
use App\Models\ImportPost;
use App\Services\ImportService;
use App\Services\StatusService;
use App\Http\Resources\ImportStatus;
use App\Follower;
use App\User;
class ImportPostController extends Controller
{
public function __construct()
{
$this->middleware('auth');
}
public function getConfig(Request $request)
{
return [
'enabled' => config('import.instagram.enabled'),
'limits' => [
'max_posts' => config('import.instagram.limits.max_posts'),
'max_attempts' => config('import.instagram.limits.max_attempts'),
],
'allow_video_posts' => config('import.instagram.allow_video_posts'),
'permissions' => [
'admins_only' => config('import.instagram.permissions.admins_only'),
'admin_follows_only' => config('import.instagram.permissions.admin_follows_only'),
'min_account_age' => config('import.instagram.permissions.min_account_age'),
'min_follower_count' => config('import.instagram.permissions.min_follower_count'),
],
'allowed' => $this->checkPermissions($request, false)
];
}
public function getProcessingCount(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
$processing = ImportPost::whereProfileId($request->user()->profile_id)
->whereNull('status_id')
->whereSkipMissingMedia(false)
->count();
$finished = ImportPost::whereProfileId($request->user()->profile_id)
->whereNotNull('status_id')
->whereSkipMissingMedia(false)
->count();
return response()->json([
'processing_count' => $processing,
'finished_count' => $finished,
]);
}
public function getImportedFiles(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
return response()->json(
ImportService::getImportedFiles($request->user()->profile_id),
200,
[],
JSON_UNESCAPED_SLASHES
);
}
public function getImportedPosts(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
return ImportStatus::collection(
ImportPost::whereProfileId($request->user()->profile_id)
->whereNotNull('status_id')
->cursorPaginate(9)
);
}
public function store(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
$this->checkPermissions($request);
$uid = $request->user()->id;
$pid = $request->user()->profile_id;
foreach($request->input('files') as $file) {
$media = $file['media'];
$c = collect($media);
$postHash = hash('sha256', $c->toJson());
$exts = $c->map(function($m) {
$fn = last(explode('/', $m['uri']));
return last(explode('.', $fn));
});
$postType = 'photo';
if($exts->count() > 1) {
if($exts->contains('mp4')) {
if($exts->contains('jpg', 'png')) {
$postType = 'photo:video:album';
} else {
$postType = 'video:album';
}
} else {
$postType = 'photo:album';
}
} else {
if(in_array($exts[0], ['jpg', 'png'])) {
$postType = 'photo';
} else if(in_array($exts[0], ['mp4'])) {
$postType = 'video';
}
}
$ip = new ImportPost;
$ip->user_id = $uid;
$ip->profile_id = $pid;
$ip->post_hash = $postHash;
$ip->service = 'instagram';
$ip->post_type = $postType;
$ip->media_count = $c->count();
$ip->media = $c->map(function($m) {
return [
'uri' => $m['uri'],
'title' => $m['title'],
'creation_timestamp' => $m['creation_timestamp']
];
})->toArray();
$ip->caption = $c->count() > 1 ? $file['title'] : $ip->media[0]['title'];
$ip->filename = last(explode('/', $ip->media[0]['uri']));
$ip->metadata = $c->map(function($m) {
return [
'uri' => $m['uri'],
'media_metadata' => isset($m['media_metadata']) ? $m['media_metadata'] : null
];
})->toArray();
$ip->creation_date = $c->count() > 1 ? now()->parse($file['creation_timestamp']) : now()->parse($media[0]['creation_timestamp']);
$ip->creation_year = now()->parse($ip->creation_date)->format('y');
$ip->creation_month = now()->parse($ip->creation_date)->format('m');
$ip->creation_day = now()->parse($ip->creation_date)->format('d');
$ip->save();
ImportService::getImportedFiles($pid, true);
ImportService::getPostCount($pid, true);
}
return [
'msg' => 'Success'
];
}
public function storeMedia(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
$this->checkPermissions($request);
$mimes = config('import.allow_video_posts') ? 'mimetypes:image/png,image/jpeg,video/mp4' : 'mimetypes:image/png,image/jpeg';
$this->validate($request, [
'file' => 'required|array|max:10',
'file.*' => [
'required',
'file',
$mimes,
'max:' . config('pixelfed.max_photo_size')
]
]);
foreach($request->file('file') as $file) {
$fileName = $file->getClientOriginalName();
$file->storeAs('imports/' . $request->user()->id . '/', $fileName);
}
ImportService::getImportedFiles($request->user()->profile_id, true);
return [
'msg' => 'Success'
];
}
protected function checkPermissions($request, $abortOnFail = true)
{
$user = $request->user();
if($abortOnFail) {
abort_unless(config('import.instagram.enabled'), 404);
}
if($user->is_admin) {
if(!$abortOnFail) {
return true;
} else {
return;
}
}
$admin = User::whereIsAdmin(true)->first();
if(config('import.instagram.permissions.admins_only')) {
if($abortOnFail) {
abort_unless($user->is_admin, 404, 'Only admins can use this feature.');
} else {
if(!$user->is_admin) {
return false;
}
}
}
if(config('import.instagram.permissions.admin_follows_only')) {
$exists = Follower::whereProfileId($admin->profile_id)
->whereFollowingId($user->profile_id)
->exists();
if($abortOnFail) {
abort_unless(
$exists,
404,
'Only admins, and accounts they follow can use this feature'
);
} else {
if(!$exists) {
return false;
}
}
}
if(config('import.instagram.permissions.min_account_age')) {
$res = $user->created_at->lt(
now()->subDays(config('import.instagram.permissions.min_account_age'))
);
if($abortOnFail) {
abort_unless(
$res,
404,
'Your account is too new to use this feature'
);
} else {
if(!$res) {
return false;
}
}
}
if(config('import.instagram.permissions.min_follower_count')) {
$res = Follower::whereFollowingId($user->profile_id)->count() >= config('import.instagram.permissions.min_follower_count');
if($abortOnFail) {
abort_unless(
$res,
404,
'You don\'t have enough followers to use this feature'
);
} else {
if(!$res) {
return false;
}
}
}
if(intval(config('import.instagram.limits.max_posts')) > 0) {
$res = ImportService::getPostCount($user->profile_id) >= intval(config('import.instagram.limits.max_posts'));
if($abortOnFail) {
abort_if(
$res,
404,
'You have reached the limit of post imports and cannot import any more posts'
);
} else {
if($res) {
return false;
}
}
}
if(intval(config('import.instagram.limits.max_attempts')) > 0) {
$res = ImportService::getAttempts($user->profile_id) >= intval(config('import.instagram.limits.max_attempts'));
if($abortOnFail) {
abort_if(
$res,
404,
'You have reached the limit of post import attempts and cannot import any more posts'
);
} else {
if($res) {
return false;
}
}
}
if(!$abortOnFail) {
return true;
}
}
}

View file

@ -0,0 +1,20 @@
<?php
namespace App\Http\Resources;
use Illuminate\Http\Request;
use Illuminate\Http\Resources\Json\JsonResource;
use App\Services\StatusService;
class ImportStatus extends JsonResource
{
/**
* Transform the resource into an array.
*
* @return array<string, mixed>
*/
public function toArray(Request $request): array
{
return StatusService::get($this->status_id, false);
}
}

17
app/Models/ImportPost.php Normal file
View file

@ -0,0 +1,17 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class ImportPost extends Model
{
use HasFactory;
protected $casts = [
'media' => 'array',
'creation_date' => 'datetime',
'metadata' => 'json'
];
}

View file

@ -0,0 +1,105 @@
<?php
namespace App\Services;
use App\Models\ImportPost;
use Cache;
class ImportService
{
const CACHE_KEY = 'pf:import-service:';
public static function getId($userId, $year, $month, $day)
{
if($userId > 999999) {
return;
}
if($year < 9 || $year > 23) {
return;
}
if($month < 1 || $month > 12) {
return;
}
if($day < 1 || $day > 31) {
return;
}
$start = 1;
$key = self::CACHE_KEY . 'getIdRange:incr:byUserId:' . $userId . ':y-' . $year . ':m-' . $month . ':d-' . $day;
$incr = Cache::increment($key, random_int(5, 19));
if($incr > 999) {
$daysInMonth = now()->parse($day . '-' . $month . '-' . $year)->daysInMonth;
if($month == 12) {
$year = $year + 1;
$month = 1;
$day = 0;
}
if($day + 1 >= $daysInMonth) {
$day = 1;
$month = $month + 1;
} else {
$day = $day + 1;
}
return self::getId($userId, $year, $month, $day);
}
$uid = str_pad($userId, 6, 0, STR_PAD_LEFT);
$year = str_pad($year, 2, 0, STR_PAD_LEFT);
$month = str_pad($month, 2, 0, STR_PAD_LEFT);
$day = str_pad($day, 2, 0, STR_PAD_LEFT);
$zone = $year . $month . $day . str_pad($incr, 3, 0, STR_PAD_LEFT);
return [
'id' => $start . $uid . $zone,
'year' => $year,
'month' => $month,
'day' => $day,
'incr' => $incr,
];
}
public static function getPostCount($profileId, $refresh = false)
{
$key = self::CACHE_KEY . 'totalPostCountByProfileId:' . $profileId;
if($refresh) {
Cache::forget($key);
}
return intval(Cache::remember($key, 21600, function() use($profileId) {
return ImportPost::whereProfileId($profileId)->whereSkipMissingMedia(false)->count();
}));
}
public static function getAttempts($profileId)
{
$key = self::CACHE_KEY . 'attemptsByProfileId:' . $profileId;
return intval(Cache::remember($key, 21600, function() use($profileId) {
return ImportPost::whereProfileId($profileId)
->whereSkipMissingMedia(false)
->get()
->groupBy(function($item) {
return $item->created_at->format('Y-m-d');
})
->count();
}));
}
public static function clearAttempts($profileId)
{
$key = self::CACHE_KEY . 'attemptsByProfileId:' . $profileId;
return Cache::forget($key);
}
public static function getImportedFiles($profileId, $refresh = false)
{
$key = self::CACHE_KEY . 'importedPostsByProfileId:' . $profileId;
if($refresh) {
Cache::forget($key);
}
return Cache::remember($key, 21600, function() use($profileId) {
return ImportPost::whereProfileId($profileId)
->get()
->map(function($ip) {
return collect($ip->media)->map(function($m) { return $m['uri']; });
})->flatten();
});
}
}

47
config/import.php Normal file
View file

@ -0,0 +1,47 @@
<?php
return [
/*
* Import from Instagram
*
* Allow users to import posts from Instagram
*
*/
'instagram' => [
'enabled' => env('PF_IMPORT_FROM_INSTAGRAM', true),
'limits' => [
// Limit max number of posts allowed to import
'max_posts' => env('PF_IMPORT_IG_MAX_POSTS', 1000),
// Limit max import attempts allowed, set to -1 for unlimited
'max_attempts' => env('PF_IMPORT_IG_MAX_ATTEMPTS', -1),
],
// Allow archived posts that will be archived upon import
'allow_archived_posts' => false,
// Allow video posts to be imported
'allow_video_posts' => env('PF_IMPORT_IG_ALLOW_VIDEO_POSTS', true),
'permissions' => [
// Limit to admin accounts only
'admins_only' => env('PF_IMPORT_IG_PERM_ADMIN_ONLY', false),
// Limit to admin accounts and local accounts they follow only
'admin_follows_only' => env('PF_IMPORT_IG_PERM_ADMIN_FOLLOWS_ONLY', false),
// Limit to accounts older than X in days
'min_account_age' => env('PF_IMPORT_IG_PERM_MIN_ACCOUNT_AGE', 1),
// Limit to accounts with a min follower count of X
'min_follower_count' => env('PF_IMPORT_IG_PERM_MIN_FOLLOWER_COUNT', 0),
// Limit to specific user ids, in comma separated format
'user_ids' => env('PF_IMPORT_IG_PERM_ONLY_USER_IDS', null),
]
]
];

View file

@ -0,0 +1,46 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('import_posts', function (Blueprint $table) {
$table->id();
$table->bigInteger('profile_id')->unsigned()->index();
$table->unsignedInteger('user_id')->index();
$table->string('service')->index();
$table->string('post_hash')->nullable()->index();
$table->string('filename')->index();
$table->tinyInteger('media_count')->unsigned();
$table->string('post_type')->nullable();
$table->text('caption')->nullable();
$table->json('media')->nullable();
$table->tinyInteger('creation_year')->unsigned()->nullable();
$table->tinyInteger('creation_month')->unsigned()->nullable();
$table->tinyInteger('creation_day')->unsigned()->nullable();
$table->tinyInteger('creation_id')->unsigned()->nullable();
$table->bigInteger('status_id')->unsigned()->nullable()->unique()->index();
$table->timestamp('creation_date')->nullable();
$table->json('metadata')->nullable();
$table->boolean('skip_missing_media')->default(false)->index();
$table->unique(['user_id', 'post_hash']);
$table->unique(['user_id', 'creation_year', 'creation_month', 'creation_day', 'creation_id'], 'import_posts_uid_phash_unique');
$table->timestamps();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('import_posts');
}
};

9144
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -36,6 +36,7 @@
"@fancyapps/fancybox": "^3.5.7",
"@trevoreyre/autocomplete-vue": "^2.2.0",
"@web3-storage/parse-link-header": "^3.1.0",
"@zip.js/zip.js": "^2.7.14",
"animate.css": "^4.1.0",
"bigpicture": "^2.6.2",
"blurhash": "^1.1.3",

Binary file not shown.

BIN
public/js/account-import.js vendored Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
public/js/daci.chunk.914d307d69fcfcd4.js vendored Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
public/js/dms.chunk.98e12cf9137ddd87.js vendored Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
public/js/home.chunk.2d93b527d492e6de.js vendored Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
public/js/installer.js vendored

Binary file not shown.

View file

@ -1 +0,0 @@
/*! @source http://purl.eligrey.com/github/canvas-toBlob.js/blob/master/canvas-toBlob.js */

Binary file not shown.

Binary file not shown.

BIN
public/js/manifest.js vendored

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
public/js/post.chunk.cd535334efc77c34.js vendored Normal file

Binary file not shown.

View file

@ -0,0 +1 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */

Binary file not shown.

Binary file not shown.

BIN
public/js/spa.js vendored

Binary file not shown.

Binary file not shown.

BIN
public/js/vendor.js vendored

Binary file not shown.

View file

@ -48,14 +48,6 @@
* Released under the MIT license
*/
/*!
* Pusher JavaScript Library v7.6.0
* https://pusher.com/
*
* Copyright 2020, Pusher
* Released under the MIT licence.
*/
/*!
* Scroll Lock v3.1.3
* https://github.com/MohammadYounes/jquery-scrollLock

Binary file not shown.

View file

@ -0,0 +1,606 @@
<template>
<div class="h-100 pf-import">
<div v-if="!loaded" class="d-flex justify-content-center align-items-center h-100">
<b-spinner />
</div>
<template v-else>
<input type="file" name="file" class="d-none" ref="zipInput" @change="zipInputChanged" />
<template v-if="page === 1">
<div class="title">
<h3 class="font-weight-bold">Import</h3>
</div>
<hr>
<section>
<p class="lead">Account Import allows you to import your data from a supported service.</p>
</section>
<section class="mt-4">
<ul class="list-group">
<li class="list-group-item d-flex justify-content-between flex-column" style="gap:1rem">
<div class="d-flex justify-content-between align-items-center" style="gap: 1rem;">
<div>
<p class="font-weight-bold mb-1">Import from Instagram</p>
<p v-if="showDisabledWarning" class="small mb-0">This feature has been disabled by the administrators.</p>
<p v-else-if="showNotAllowedWarning" class="small mb-0">You have not been permitted to use this feature, or have reached the maximum limits. For more info, view the <a href="/site/kb/import" class="font-weight-bold">Import Help Center</a> page.</p>
<p v-else class="small mb-0">Upload the JSON export from Instagram in .zip format.<br />For more information click <a href="/site/kb/import">here</a>.</p>
</div>
<div v-if="!showDisabledWarning && !showNotAllowedWarning">
<button
v-if="step === 1 || invalidArchive"
type="button"
class="font-weight-bold btn btn-primary rounded-pill px-4 btn-lg"
@click="selectArchive()"
:disabled="showDisabledWarning">
Import
</button>
<template v-else-if="step === 2">
<div class="d-flex justify-content-center align-items-center flex-column">
<b-spinner v-if="showUploadLoader" small />
<button v-else type="button" class="font-weight-bold btn btn-outline-primary btn-sm btn-block" @click="reviewImports()">Review Imports</button>
<p v-if="zipName" class="small font-weight-bold mt-2 mb-0">{{ zipName }}</p>
</div>
</template>
</div>
</div>
</li>
</ul>
<ul class="list-group mt-3">
<li v-if="processingCount" class="list-group-item d-flex justify-content-between flex-column" style="gap:1rem">
<div class="d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-1">Processing Imported Posts</p>
<p class="small mb-0">These are posts that are in the process of being imported.</p>
</div>
<div>
<span class="btn btn-danger rounded-pill py-0 font-weight-bold" disabled>{{ processingCount }}</span>
</div>
</div>
</li>
<li v-if="finishedCount" class="list-group-item d-flex justify-content-between flex-column" style="gap:1rem">
<div class="d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-1">Imported Posts</p>
<p class="small mb-0">These are posts that have been successfully imported.</p>
</div>
<div>
<button
type="button"
class="font-weight-bold btn btn-primary btn-sm rounded-pill px-4 btn-block"
@click="handleReviewPosts()"
:disabled="!finishedCount">
Review {{ finishedCount }} Posts
</button>
</div>
</div>
</li>
</ul>
</section>
</template>
<template v-else-if="page === 2">
<div class="d-flex justify-content-between align-items-center">
<div class="title">
<h3 class="font-weight-bold">Import from Instagram</h3>
</div>
<button
class="btn btn-primary font-weight-bold rounded-pill px-4"
:class="{ disabled: !selectedMedia || !selectedMedia.length }"
:disabled="!selectedMedia || !selectedMedia.length || importButtonLoading"
@click="handleImport()"
>
<b-spinner v-if="importButtonLoading" small />
<span v-else>Import</span>
</button>
</div>
<hr>
<section>
<div class="d-flex justify-content-between align-items-center mb-3">
<div v-if="!selectedMedia || !selectedMedia.length">
<p class="lead mb-0">Review posts you'd like to import.</p>
<p class="small text-muted mb-0">Tap on posts to include them in your import.</p>
</div>
<p v-else class="lead mb-0"><span class="font-weight-bold">{{ selectedPostsCounter }}</span> posts selected for import</p>
</div>
</section>
<section class="row mb-n5 media-selector" style="max-height: 600px;overflow-y: auto;">
<div v-for="media in postMeta" class="col-12 col-md-4">
<div
class="square cursor-pointer"
@click="toggleSelectedPost(media)">
<div
v-if="media.media[0].uri.endsWith('.mp4')"
:class="{ selected: selectedMedia.indexOf(media.media[0].uri) != -1 }"
class="info-overlay-text-label rounded">
<h5 class="text-white m-auto font-weight-bold">
<span>
<span class="far fa-video fa-2x p-2 d-flex-inline"></span>
</span>
</h5>
</div>
<div
v-else
class="square-content"
:class="{ selected: selectedMedia.indexOf(media.media[0].uri) != -1 }"
:style="{ borderRadius: '5px', backgroundImage: 'url(' + getFileNameUrl(media.media[0].uri) + ')'}">
</div>
</div>
<div class="d-flex mt-1 justify-content-between align-items-center">
<p class="small"><i class="far fa-clock"></i> {{ formatDate(media.media[0].creation_timestamp) }}</p>
<p class="small font-weight-bold"><a href="#" @click.prevent="showDetailsModal(media)"><i class="far fa-info-circle"></i> Details</a></p>
</div>
</div>
</section>
</template>
<template v-else-if="page === 'reviewImports'">
<div class="d-flex justify-content-between align-items-center">
<div class="title">
<h3 class="font-weight-bold">Posts Imported from Instagram</h3>
</div>
</div>
<hr>
<section class="row mb-n5 media-selector" style="max-height: 600px;overflow-y: auto;">
<div v-for="media in importedPosts.data" class="col-12 col-md-4">
<div
class="square cursor-pointer">
<div
v-if="media.media_attachments[0].url.endsWith('.mp4')"
class="info-overlay-text-label rounded">
<h5 class="text-white m-auto font-weight-bold">
<span>
<span class="far fa-video fa-2x p-2 d-flex-inline"></span>
</span>
</h5>
</div>
<div
v-else
class="square-content"
:style="{ borderRadius: '5px', backgroundImage: 'url(' + media.media_attachments[0].url + ')'}">
</div>
</div>
<div class="d-flex mt-1 justify-content-between align-items-center">
<p class="small"><i class="far fa-clock"></i> {{ formatDate(media.created_at, false) }}</p>
<p class="small font-weight-bold"><a :href="media.url"><i class="far fa-info-circle"></i> View</a></p>
</div>
</div>
<div class="col-12 my-3">
<button
v-if="importedPosts.meta && importedPosts.meta.next_cursor"
class="btn btn-primary btn-block font-weight-bold"
@click="loadMorePosts()">
Load more
</button>
</div>
</section>
</template>
</template>
<b-modal
id="detailsModal"
title="Post Details"
v-model="detailsModalShow"
:ok-only="true"
ok-title="Close"
centered>
<div class="">
<div v-for="(media, idx) in modalData.media" class="mb-3">
<div class="list-group">
<div class="list-group-item d-flex justify-content-between align-items-center">
<p class="text-center font-weight-bold mb-0">Media #{{idx + 1}}</p>
<img :src="getFileNameUrl(media.uri)" width="30" height="30" style="object-fit: cover; border-radius: 5px;">
</div>
<div class="list-group-item">
<p class="small text-muted">Caption</p>
<p class="mb-0 small read-more" style="font-size: 12px;overflow-y: hidden;">{{ media.title ? media.title : modalData.title }}</p>
</div>
<div class="list-group-item">
<div class="d-flex justify-content-between align-items-center">
<p class="small mb-0 text-muted">Timestamp</p>
<p class="font-weight-bold mb-0">{{ formatDate(media.creation_timestamp) }}</p>
</div>
</div>
</div>
</div>
</div>
</b-modal>
</div>
</template>
<script type="text/javascript">
import * as zip from "@zip.js/zip.js";
export default {
data() {
return {
page: 1,
step: 1,
toggleLimit: 100,
config: {},
showDisabledWarning: false,
showNotAllowedWarning: false,
invalidArchive: false,
loaded: false,
existing: [],
zipName: undefined,
zipFiles: [],
postMeta: [],
imageCache: [],
includeArchives: false,
selectedMedia: [],
selectedPostsCounter: 0,
detailsModalShow: false,
modalData: {},
importedPosts: [],
finishedCount: undefined,
processingCount: undefined,
showUploadLoader: false,
importButtonLoading: false,
}
},
mounted() {
this.fetchConfig();
},
methods: {
fetchConfig() {
axios.get('/api/local/import/ig/config')
.then(res => {
this.config = res.data;
if(res.data.enabled == false) {
this.showDisabledWarning = true;
this.loaded = true;
} else if(res.data.allowed == false) {
this.showNotAllowedWarning = true;
this.loaded = true;
} else {
this.fetchExisting();
}
})
},
fetchExisting() {
axios.post('/api/local/import/ig/existing')
.then(res => {
this.existing = res.data;
})
.finally(() => {
this.fetchProcessing();
})
},
fetchProcessing() {
axios.post('/api/local/import/ig/processing')
.then(res => {
this.processingCount = res.data.processing_count;
this.finishedCount = res.data.finished_count;
})
.finally(() => {
this.loaded = true;
})
},
selectArchive() {
event.currentTarget.blur();
swal({
title: 'Upload Archive',
icon: 'success',
text: 'The .zip archive is probably named something like username_20230606.zip, and was downloaded from the Instagram.com website.',
buttons: {
cancel: "Cancel",
danger: {
text: "Upload zip archive",
value: "upload"
}
}
})
.then(res => {
this.$refs.zipInput.click();
})
},
zipInputChanged(event) {
this.step = 2;
this.zipName = event.target.files[0].name;
this.showUploadLoader = true;
setTimeout(() => {
this.reviewImports();
}, 1000);
setTimeout(() => {
this.showUploadLoader = false;
}, 3000);
},
reviewImports() {
this.invalidArchive = false;
this.checkZip();
},
model(file, options = {}) {
return (new zip.ZipReader(new zip.BlobReader(file))).getEntries(options);
},
formatDate(ts, unixt = true) {
let date = unixt ? new Date(ts * 1000) : new Date(ts);
return date.toLocaleDateString()
},
getFileNameUrl(filename) {
return this.imageCache.filter(e => e.filename === filename).map(e => e.blob);
},
showDetailsModal(entry) {
this.modalData = entry;
this.detailsModalShow = true;
setTimeout(() => {
pixelfed.readmore();
}, 500);
},
filterPostMeta(media) {
let json = JSON.parse(media);
let res = json.filter(j => {
let ids = j.media.map(m => m.uri).filter(m => {
if(this.config.allow_video_posts == true) {
return m.endsWith('.png') || m.endsWith('.jpg') || m.endsWith('.mp4');
} else {
return m.endsWith('.png') || m.endsWith('.jpg');
}
});
return ids.length;
}).filter(j => {
let ids = j.media.map(m => m.uri);
return !this.existing.includes(ids[0]);
})
this.postMeta = res;
return res;
},
async checkZip() {
let file = this.$refs.zipInput.files[0];
let entries = await this.model(file);
if (entries && entries.length) {
let files = await entries.filter(e => e.filename === 'content/posts_1.json');
if(!files || !files.length) {
this.contactModal(
'Invalid import archive',
"The .zip archive you uploaded is corrupted, or is invalid. We cannot process your import at this time.\n\nIf this issue persists, please contact an administrator.",
'error'
)
this.invalidArchive = true;
return;
} else {
this.readZip();
}
}
},
async readZip() {
let file = this.$refs.zipInput.files[0];
let entries = await this.model(file);
if (entries && entries.length) {
this.zipFiles = entries;
let media = await entries.filter(e => e.filename === 'content/posts_1.json')[0].getData(new zip.TextWriter());
this.filterPostMeta(media);
let imgs = await Promise.all(entries.filter(entry => {
return entry.filename.startsWith('media/posts/') && (entry.filename.endsWith('.png') || entry.filename.endsWith('.jpg') || entry.filename.endsWith('.mp4'));
})
.map(async entry => {
if(
entry.filename.startsWith('media/posts/') &&
(
entry.filename.endsWith('.png') ||
entry.filename.endsWith('.jpg') ||
entry.filename.endsWith('.mp4')
)
) {
let types = {
'png': 'image/png',
'jpg': 'image/jpeg',
'jpeg': 'image/jpeg',
'mp4': 'video/mp4'
}
let type = types[entry.filename.split('/').pop().split('.').pop()];
let blob = await entry.getData(new zip.BlobWriter(type));
let url = URL.createObjectURL(blob);
return {
filename: entry.filename,
blob: url,
file: blob
}
} else {
return;
}
}));
this.imageCache = imgs.flat(2);
}
setTimeout(() => {
this.page = 2;
}, 500);
},
toggleLimitReached() {
this.contactModal(
'Limit reached',
"You can only import " + this.toggleLimit + " posts at a time.\nYou can import more posts after you finish importing these posts.",
'error'
)
},
toggleSelectedPost(media) {
let filename;
let self = this;
if(media.media.length === 1) {
filename = media.media[0].uri
if(this.selectedMedia.indexOf(filename) == -1) {
if(this.selectedPostsCounter >= this.toggleLimit) {
this.toggleLimitReached();
return;
}
this.selectedMedia.push(filename);
this.selectedPostsCounter++;
} else {
let idx = this.selectedMedia.indexOf(filename);
this.selectedMedia.splice(idx, 1);
this.selectedPostsCounter--;
}
} else {
filename = media.media[0].uri
if(this.selectedMedia.indexOf(filename) == -1) {
if(this.selectedPostsCounter >= this.toggleLimit) {
this.toggleLimitReached();
return;
}
this.selectedPostsCounter++;
} else {
this.selectedPostsCounter--;
}
media.media.forEach(function(m) {
filename = m.uri
if(self.selectedMedia.indexOf(filename) == -1) {
self.selectedMedia.push(filename);
} else {
let idx = self.selectedMedia.indexOf(filename);
self.selectedMedia.splice(idx, 1);
}
})
}
},
sliceIntoChunks(arr, chunkSize) {
const res = [];
for (let i = 0; i < arr.length; i += chunkSize) {
const chunk = arr.slice(i, i + chunkSize);
res.push(chunk);
}
return res;
},
handleImport() {
swal('Importing...', "Please wait while we upload your imported posts.\n Keep this page open and do not navigate away.", 'success');
this.importButtonLoading = true;
let ic = this.imageCache.filter(e => {
return this.selectedMedia.indexOf(e.filename) != -1;
})
let chunks = this.sliceIntoChunks(ic, 10);
chunks.forEach(c => {
let formData = new FormData();
c.map((e, idx) => {
let file = new File([e.file], e.filename);
formData.append('file['+ idx +']', file, e.filename.split('/').pop());
})
axios.post(
'/api/local/import/ig/media',
formData,
{
headers: {
'Content-Type': `multipart/form-data`,
},
}
)
.catch(err => {
this.contactModal(
'Error',
err.response.data.message,
'error'
)
});
})
axios.post('/api/local/import/ig', {
files: this.postMeta.filter(e => this.selectedMedia.includes(e.media[0].uri)).map(e => {
if(e.hasOwnProperty('title')) {
return {
title: e.title,
'creation_timestamp': e.creation_timestamp,
uri: e.uri,
media: e.media
}
} else {
return {
title: null,
'creation_timestamp': null,
uri: null,
media: e.media
}
}
})
}).then(res => {
if(res) {
setTimeout(() => {
window.location.reload()
}, 5000);
}
}).catch(err => {
this.contactModal(
'Error',
err.response.data.error,
'error'
)
})
},
handleReviewPosts() {
this.page = 'reviewImports';
axios.post('/api/local/import/ig/posts')
.then(res => {
this.importedPosts = res.data;
})
},
loadMorePosts() {
event.currentTarget.blur();
axios.post('/api/local/import/ig/posts', {
cursor: this.importedPosts.meta.next_cursor
})
.then(res => {
let data = res.data;
data.data = [...this.importedPosts.data, ...res.data.data];
this.importedPosts = data;
})
},
contactModal(title = 'Error', text, icon, closeButton = 'Close') {
swal({
title: title,
text: text,
icon: icon,
dangerMode: true,
buttons: {
ok: closeButton,
danger: {
text: 'Contact Support',
value: 'contact'
}
}
})
.then(res => {
if(res === 'contact') {
window.location.href = '/site/contact'
}
});
}
}
}
</script>
<style lang="scss" scoped>
.pf-import {
.media-selector {
.selected {
border: 5px solid red;
}
}
}
</style>

4
resources/assets/js/account-import.js vendored Normal file
View file

@ -0,0 +1,4 @@
Vue.component(
'account-import',
require('./../components/AccountImport.vue').default
);

View file

@ -1,27 +1,10 @@
@extends('settings.template')
@section('section')
<div class="title">
<h3 class="font-weight-bold">Import</h3>
</div>
<hr>
<section>
<p class="lead">Account Import allows you to import your data from a supported service. <a href="#">Learn more.</a></p>
<p class="alert alert-warning"><strong>Warning: </strong> Imported posts will not appear on timelines or be delivered to followers.</p>
</section>
<section class="mt-4">
<p class="small text-muted font-weight-bold text-uppercase mb-3">Supported Services</p>
<p class="">
<a class="btn btn-outline-primary font-weight-bold" href="{{route('settings.import.ig')}}">Import from Instagram</a>
</p>
<hr>
<p class="small text-muted font-weight-bold text-uppercase mb-3">Coming Soon</p>
<p class="">
<a class="btn btn-outline-secondary font-weight-bold disabled" href="#">Import from Pixelfed</a>
</p>
<p class="">
<a class="btn btn-outline-secondary font-weight-bold disabled" href="#">Import from Mastodon</a>
</p>
</section>
<account-import />
@endsection
@push('scripts')
<script type="text/javascript" src="{{ mix('js/account-import.js') }}"></script>
@endpush

View file

@ -254,6 +254,22 @@
</div>
</a>
</div> --}}
<div class="col-12 col-md-6 mb-3">
<a href="{{route('help.import')}}" class="text-decoration-none">
<div class="card">
<div class="card-body">
<p class="py-1 text-center">
<i class="far fa-file-import text-lighter fa-2x"></i>
</p>
<p class="text-center text-muted font-weight-bold h4 mb-0">Import</p>
<div class="text-center pt-3">
<p class="small text-dark font-weight-bold mb-0">How to Import from Instagram</p>
<p class="small text-dark font-weight-bold mb-0">Troubleshooting Imports</p>
</div>
</div>
</div>
</a>
</div>
</div>
@endsection

View file

@ -0,0 +1,94 @@
@extends('site.help.partial.template', ['breadcrumb'=>'Import'])
@section('section')
<div class="title">
<h3 class="font-weight-bold">Import</h3>
</div>
<hr>
<p class="lead py-3">With the Import from Instagram feature, you can seamlessly transfer your photos, captions, and even hashtags from your Instagram account to Pixelfed, ensuring a smooth transition without losing your cherished memories or creative expressions.</p>
<hr class="mb-4" />
<p class="text-center font-weight-bold">How to get your export data from Instagram:</p>
<ol class="pb-4">
<li class="mb-2">
<span>Follow the Instagram instructions on <strong>Downloading a copy of your data on Instagram</strong> on <a href="https://help.instagram.com/181231772500920" class="font-weight-bold">this page</a>. <strong class="text-danger small font-weight-bold">Make sure you select the JSON format</strong></span>
</li>
<li class="mb-2">
<span>Wait for the email from Instagram with your download link</span>
</li>
<li class="mb-2">
<span>Download your .zip export from Instagram</span>
</li>
<li class="mb-2">
<span>Navigate to the <a href="/settings/import" class="font-weight-bold">Import</a> settings page</span>
</li>
<li class="">
<span>Follow the instructions and import your posts 🥳</span>
</li>
</ol>
<hr class="mb-4" />
<p class="text-center font-weight-bold">Import Limits</p>
<div class="list-group pb-4">
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Max Posts</p>
<p class="small mb-0">The maximum imported posts allowed</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.limits.max_posts') == -1 ? 'Unlimited' : config('import.instagram.limits.max_posts') }}</div>
</div>
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Max Attempts</p>
<p class="small mb-0">The maximum import attempts allowed<br />(counted as total imports grouped by day)</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.limits.max_attempts') == -1 ? 'Unlimited' : config('import.instagram.limits.max_attempts') }}</div>
</div>
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Video Imports</p>
<p class="small mb-0">The server supports importing video posts</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.allow_video_posts') ? '✅' : '❌' }}</div>
</div>
</div>
<hr class="mb-4" />
<p class="text-center font-weight-bold mb-0">Import Permissions</p>
<p class="text-center small">Who is allowed to use the Import feature</p>
<div class="list-group">
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Only Admins</p>
<p class="small mb-0">Only admin accounts can import</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.permissions.admins_only') ? '✅' : '❌' }}</div>
</div>
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Only Admins + Following</p>
<p class="small mb-0">Only admin accounts, or accounts they follow, can import</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.permissions.admin_follows_only') ? '✅' : '❌' }}</div>
</div>
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Minimum Account Age</p>
<p class="small mb-0">Only accounts with a minimum age in days can import</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.permissions.min_account_age')}}</div>
</div>
<div class="list-group-item d-flex justify-content-between align-items-center">
<div>
<p class="font-weight-bold mb-0">Minimum Follower Count</p>
<p class="small mb-0">Only accounts with a minimum follower count can import</p>
</div>
<div class="font-weight-bold">{{ config('import.instagram.permissions.min_follower_count')}}</div>
</div>
</div>
@endsection

View file

@ -21,12 +21,18 @@
{{-- <li class="nav-item {{request()->is('*/direct-messages')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.dm')}}">{{__('helpcenter.directMessages')}}</a>
</li> --}}
{{-- <li class="nav-item {{request()->is('*/tagging-people')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.tagging-people')}}">{{__('helpcenter.taggingPeople')}}</a>
</li> --}}
<li class="nav-item {{request()->is('*/timelines')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.timelines')}}">{{__('helpcenter.timelines')}}</a>
</li>
{{-- <li class="nav-item {{request()->is('*/embed')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.embed')}}">{{__('helpcenter.embed')}}</a>
</li> --}}
<li class="nav-item {{request()->is('*/import')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.import')}}">Instagram Import</a>
</li>
<li class="nav-item">
<hr>
</li>
@ -37,13 +43,13 @@
</li>
{{-- <li class="nav-item {{request()->is('*/what-is-the-fediverse')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.what-is-fediverse')}}">{{__('helpcenter.whatIsTheFediverse')}}</a>
</li> --}}
{{-- <li class="nav-item {{request()->is('*/controlling-visibility')?'active':''}}">
</li>
<li class="nav-item {{request()->is('*/controlling-visibility')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.controlling-visibility')}}">
{{__('helpcenter.controllingVisibility')}}
</a>
</li> --}}
{{-- <li class="nav-item {{request()->is('*/blocking-accounts')?'active':''}}">
</li>
<li class="nav-item {{request()->is('*/blocking-accounts')?'active':''}}">
<a class="nav-link font-weight-light text-muted" href="{{route('help.blocking-accounts')}}">
{{__('helpcenter.blockingAccounts')}}
</a>

View file

@ -306,6 +306,13 @@ Route::domain(config('pixelfed.domain.app'))->middleware(['validemail', 'twofact
Route::get('profile/collections/{id}', 'CollectionController@getUserCollections');
Route::post('compose/tag/untagme', 'MediaTagController@untagProfile');
Route::post('import/ig', 'ImportPostController@store');
Route::get('import/ig/config', 'ImportPostController@getConfig');
Route::post('import/ig/media', 'ImportPostController@storeMedia');
Route::post('import/ig/existing', 'ImportPostController@getImportedFiles');
Route::post('import/ig/posts', 'ImportPostController@getImportedPosts');
Route::post('import/ig/processing', 'ImportPostController@getProcessingCount');
});
Route::group(['prefix' => 'web/stories'], function () {
@ -577,6 +584,7 @@ Route::domain(config('pixelfed.domain.app'))->middleware(['validemail', 'twofact
Route::view('tagging-people', 'site.help.tagging-people')->name('help.tagging-people');
Route::view('licenses', 'site.help.licenses')->name('help.licenses');
Route::view('instance-max-users-limit', 'site.help.instance-max-users')->name('help.instance-max-users-limit');
Route::view('import', 'site.help.import')->name('help.import');
});
Route::get('newsroom/{year}/{month}/{slug}', 'NewsroomController@show');
Route::get('newsroom/archive', 'NewsroomController@archive');

1
webpack.mix.js vendored
View file

@ -34,6 +34,7 @@ mix.js('resources/assets/js/app.js', 'public/js')
.js('resources/assets/js/spa.js', 'public/js')
.js('resources/assets/js/stories.js', 'public/js')
.js('resources/assets/js/portfolio.js', 'public/js')
.js('resources/assets/js/account-import.js', 'public/js')
.js('resources/assets/js/admin_invite.js', 'public/js')
.js('resources/assets/js/landing.js', 'public/js')
.vue({ version: 2 });