merge dev

This commit is contained in:
Christian Winther 2024-01-04 11:33:54 +00:00
commit 7b3e11012f
248 changed files with 17043 additions and 8376 deletions

View file

@ -1,8 +1,8 @@
root = true
[*]
indent_style = space
indent_size = 4
indent_style = tab
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true

View file

@ -1,6 +1,89 @@
# Release Notes
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
### Added
- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
- Video WebP2P ([#4713](https://github.com/pixelfed/pixelfed/pull/4713)) ([0405ef12](https://github.com/pixelfed/pixelfed/commit/0405ef12))
- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
- Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
- Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
- Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
- Added Mutual Followers API endpoint ([33dbbe46](https://github.com/pixelfed/pixelfed/commit/33dbbe46))
- Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac))
### Federation
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
- ([](https://github.com/pixelfed/pixelfed/commit/))
### Updates
- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59))
- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904))
- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce))
- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d))
- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec))
- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d))
- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973))
- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7))
- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605))
- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27))
- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed))
- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3))
- Update StatusTransformer, generate autolink on request ([dfe2379b](https://github.com/pixelfed/pixelfed/commit/dfe2379b))
- Update ComposeModal component, fix multi filter bug and allow media re-ordering before upload/posting ([56e315f6](https://github.com/pixelfed/pixelfed/commit/56e315f6))
- Update ApiV1Dot1Controller, allow iar rate limits to be configurable ([28a80803](https://github.com/pixelfed/pixelfed/commit/28a80803))
- Update ApiV1Dot1Controller, add domain to iar redirect ([1f82d47c](https://github.com/pixelfed/pixelfed/commit/1f82d47c))
- Update ApiV1Dot1Controller, add configurable app confirm rate limit ttl ([4c6a0719](https://github.com/pixelfed/pixelfed/commit/4c6a0719))
- Update LikePipeline, dispatch to feed queue. Fixes ([#4723](https://github.com/pixelfed/pixelfed/issues/4723)) ([da510089](https://github.com/pixelfed/pixelfed/commit/da510089))
- Update AccountImport ([5a2d7e3e](https://github.com/pixelfed/pixelfed/commit/5a2d7e3e))
- Update ImportPostController, fix IG bug with missing spaces between hashtags ([9c24157a](https://github.com/pixelfed/pixelfed/commit/9c24157a))
- Update ApiV1Controller, fix mutes in home feed ([ddc21714](https://github.com/pixelfed/pixelfed/commit/ddc21714))
- Update AP helpers, improve preferredUsername validation ([21218c79](https://github.com/pixelfed/pixelfed/commit/21218c79))
- Update delete pipelines, properly invoke StatusHashtag delete events ([ce54d29c](https://github.com/pixelfed/pixelfed/commit/ce54d29c))
- Update mail config ([0e431271](https://github.com/pixelfed/pixelfed/commit/0e431271))
- Update hashtag following ([015b1b80](https://github.com/pixelfed/pixelfed/commit/015b1b80))
- Update IncrementPostCount job, prevent overlap ([b2c9cc23](https://github.com/pixelfed/pixelfed/commit/b2c9cc23))
- Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885))
- Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae))
- Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008))
- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
- Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
- Update StoryApiV1Controller, add self-carousel endpoint. Fixes ([#4352](https://github.com/pixelfed/pixelfed/issues/4352)) ([bcb88d5b](https://github.com/pixelfed/pixelfed/commit/bcb88d5b))
- Update FollowServiceWarmCache, use more efficient query ([fe9b4c5a](https://github.com/pixelfed/pixelfed/commit/fe9b4c5a))
- Update HomeFeedPipeline, observe mutes/blocks during fanout ([8548294c](https://github.com/pixelfed/pixelfed/commit/8548294c))
- Update FederationController, add proper following/follower counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
- Update FederationController, add proper statuses counts ([3204fb96](https://github.com/pixelfed/pixelfed/commit/3204fb96))
- Update Inbox handler, fix missing object_url and uri fields for direct statuses ([a0157fce](https://github.com/pixelfed/pixelfed/commit/a0157fce))
- Update DirectMessageController, deliver direct delete activities to user inbox instead of sharedInbox ([d848792a](https://github.com/pixelfed/pixelfed/commit/d848792a))
- Update DirectMessageController, dispatch deliver and delete actions to the job queue ([7f462a80](https://github.com/pixelfed/pixelfed/commit/7f462a80))
- Update Inbox, improve story attribute collection ([06bee36c](https://github.com/pixelfed/pixelfed/commit/06bee36c))
- Update DirectMessageController, dispatch local deletes to pipeline ([98186564](https://github.com/pixelfed/pixelfed/commit/98186564))
- Update StatusPipeline, fix Direct and Story notification deletion ([4c95306f](https://github.com/pixelfed/pixelfed/commit/4c95306f))
- Update Notifications.vue, fix deprecated DM action links for story activities ([4c3823b0](https://github.com/pixelfed/pixelfed/commit/4c3823b0))
- Update ComposeModal, fix missing alttext post state ([0a068119](https://github.com/pixelfed/pixelfed/commit/0a068119))
- Update PhotoAlbumPresenter.vue, fix fullscreen mode ([822e9888](https://github.com/pixelfed/pixelfed/commit/822e9888))
- Update Timeline.vue, improve CHT pagination ([9c43e7e2](https://github.com/pixelfed/pixelfed/commit/9c43e7e2))
- Update HomeFeedPipeline, fix StatusService validation ([041c0135](https://github.com/pixelfed/pixelfed/commit/041c0135))
- Update Inbox, improve tombstone query efficiency ([759a4393](https://github.com/pixelfed/pixelfed/commit/759a4393))
- Update AccountService, add setLastActive method ([ebbd98e7](https://github.com/pixelfed/pixelfed/commit/ebbd98e7))
- Update ApiV1Controller, set last_active_at ([b6419545](https://github.com/pixelfed/pixelfed/commit/b6419545))
- Update AdminShadowFilter, fix deleted profile bug ([a492a95a](https://github.com/pixelfed/pixelfed/commit/a492a95a))
- Update FollowerService, add $silent param to remove method to more efficently purge relationships ([1664a5bc](https://github.com/pixelfed/pixelfed/commit/1664a5bc))
- Update AP ProfileTransformer, add published attribute ([adfaa2b1](https://github.com/pixelfed/pixelfed/commit/adfaa2b1))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

View file

@ -0,0 +1,106 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\User;
use App\Models\DefaultDomainBlock;
use App\Models\UserDomainBlock;
use function Laravel\Prompts\text;
use function Laravel\Prompts\confirm;
use function Laravel\Prompts\progress;
class AddUserDomainBlock extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:add-user-domain-block';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Apply a domain block to all users';
/**
* Execute the console command.
*/
public function handle()
{
$domain = text('Enter domain you want to block');
$domain = strtolower($domain);
$domain = $this->validateDomain($domain);
if(!$domain || empty($domain)) {
$this->error('Invalid domain');
return;
}
$this->processBlocks($domain);
return;
}
protected function validateDomain($domain)
{
if(!strpos($domain, '.')) {
return;
}
if(str_starts_with($domain, 'https://')) {
$domain = str_replace('https://', '', $domain);
}
if(str_starts_with($domain, 'http://')) {
$domain = str_replace('http://', '', $domain);
}
$domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
$valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
if(!$valid) {
return;
}
if($domain === config('pixelfed.domain.app')) {
$this->error('Invalid domain');
return;
}
$confirmed = confirm('Are you sure you want to block ' . $domain . '?');
if(!$confirmed) {
return;
}
return $domain;
}
protected function processBlocks($domain)
{
DefaultDomainBlock::updateOrCreate([
'domain' => $domain
]);
progress(
label: 'Updating user domain blocks...',
steps: User::lazyById(500),
callback: fn ($user) => $this->performTask($user, $domain),
);
}
protected function performTask($user, $domain)
{
if(!$user->profile_id || $user->delete_after) {
return;
}
if($user->status != null && $user->status != 'disabled') {
return;
}
UserDomainBlock::updateOrCreate([
'profile_id' => $user->profile_id,
'domain' => $domain
]);
}
}

View file

@ -0,0 +1,115 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Cache;
use Storage;
use App\Avatar;
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
class AvatarStorageDeepClean extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'avatar:storage-deep-clean';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleanup avatar storage';
protected $shouldKeepRunning = true;
protected $counter = 0;
/**
* Execute the console command.
*/
public function handle(): void
{
$this->info(' ____ _ ______ __ ');
$this->info(' / __ \(_) _____ / / __/__ ____/ / ');
$this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / ');
$this->info(' / ____/ /> </ __/ / __/ __/ /_/ / ');
$this->info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ ');
$this->info(' ');
$this->info(' Pixelfed Avatar Deep Cleaner');
$this->line(' ');
$this->info(' Purge/delete old and outdated avatars from remote accounts');
$this->line(' ');
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
$this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
exit;
}
$start = 0;
if(!$this->confirm('Are you sure you want to proceed?')) {
$this->error('Aborting...');
exit;
}
if(!$this->activeCheck()) {
$this->info('Found existing deep cleaning job');
if(!$this->confirm('Do you want to continue where you left off?')) {
$this->error('Aborting...');
exit;
} else {
$start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
if($start && $start < 1 || $start > PHP_INT_MAX) {
$this->error('Error fetching cached value');
$this->error('Aborting...');
exit;
}
}
}
$count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
$bar = $this->output->createProgressBar($count);
foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
usleep(random_int(50, 1000));
$this->counter++;
$this->handleAvatar($avatar);
$bar->advance();
}
$bar->finish();
}
protected function updateCache($id)
{
Cache::put('cmd:asdp', $id);
if($this->counter % 5 === 0) {
Storage::put('avatar-deep-clean.json', $id);
}
}
protected function activeCheck()
{
if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
return false;
}
return true;
}
protected function handleAvatar($avatar)
{
$this->updateCache($avatar->id);
$queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
$queue = $queues[random_int(0, 7)];
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
}
}

View file

@ -0,0 +1,96 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\User;
use App\Models\DefaultDomainBlock;
use App\Models\UserDomainBlock;
use function Laravel\Prompts\text;
use function Laravel\Prompts\confirm;
use function Laravel\Prompts\progress;
class DeleteUserDomainBlock extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:delete-user-domain-block';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Remove a domain block for all users';
/**
* Execute the console command.
*/
public function handle()
{
$domain = text('Enter domain you want to unblock');
$domain = strtolower($domain);
$domain = $this->validateDomain($domain);
if(!$domain || empty($domain)) {
$this->error('Invalid domain');
return;
}
$this->processUnblocks($domain);
return;
}
protected function validateDomain($domain)
{
if(!strpos($domain, '.')) {
return;
}
if(str_starts_with($domain, 'https://')) {
$domain = str_replace('https://', '', $domain);
}
if(str_starts_with($domain, 'http://')) {
$domain = str_replace('http://', '', $domain);
}
$domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST));
$valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE);
if(!$valid) {
return;
}
if($domain === config('pixelfed.domain.app')) {
return;
}
$confirmed = confirm('Are you sure you want to unblock ' . $domain . '?');
if(!$confirmed) {
return;
}
return $domain;
}
protected function processUnblocks($domain)
{
DefaultDomainBlock::whereDomain($domain)->delete();
if(!UserDomainBlock::whereDomain($domain)->count()) {
$this->info('No results found!');
return;
}
progress(
label: 'Updating user domain blocks...',
steps: UserDomainBlock::whereDomain($domain)->lazyById(500),
callback: fn ($domainBlock) => $this->performTask($domainBlock),
);
}
protected function performTask($domainBlock)
{
$domainBlock->deleteQuietly();
}
}

View file

@ -0,0 +1,57 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Hashtag;
use App\StatusHashtag;
use DB;
class HashtagCachedCountUpdate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:hashtag-cached-count-update {--limit=100}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Update cached counter of hashtags';
/**
* Execute the console command.
*/
public function handle()
{
$limit = $this->option('limit');
$tags = Hashtag::whereNull('cached_count')->limit($limit)->get();
$count = count($tags);
if(!$count) {
return;
}
$bar = $this->output->createProgressBar($count);
$bar->start();
foreach($tags as $tag) {
$count = DB::table('status_hashtags')->whereHashtagId($tag->id)->count();
if(!$count) {
$tag->cached_count = 0;
$tag->saveQuietly();
$bar->advance();
continue;
}
$tag->cached_count = $count;
$tag->saveQuietly();
$bar->advance();
}
$bar->finish();
$this->line(' ');
return;
}
}

View file

@ -0,0 +1,94 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Hashtag;
use App\StatusHashtag;
use App\Models\HashtagRelated;
use App\Services\HashtagRelatedService;
use Illuminate\Contracts\Console\PromptsForMissingInput;
use function Laravel\Prompts\multiselect;
use function Laravel\Prompts\confirm;
class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:hashtag-related-generate {tag}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Prompt for missing input arguments using the returned questions.
*
* @return array
*/
protected function promptForMissingArgumentsUsing()
{
return [
'tag' => 'Which hashtag should we generate related tags for?',
];
}
/**
* Execute the console command.
*/
public function handle()
{
$tag = $this->argument('tag');
$hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
if(!$hashtag) {
$this->error('Hashtag not found, aborting...');
exit;
}
$exists = HashtagRelated::whereHashtagId($hashtag->id)->exists();
if($exists) {
$confirmed = confirm('Found existing related tags, do you want to regenerate them?');
if(!$confirmed) {
$this->error('Aborting...');
exit;
}
}
$this->info('Looking up #' . $tag . '...');
$tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
if(!$tags || $tags < 100) {
$this->error('Not enough posts found to generate related hashtags!');
exit;
}
$this->info('Found ' . $tags . ' posts that use that hashtag');
$related = collect(HashtagRelatedService::fetchRelatedTags($tag));
$selected = multiselect(
label: 'Which tags do you want to generate?',
options: $related->pluck('name'),
required: true,
);
$filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
$agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
HashtagRelated::updateOrCreate([
'hashtag_id' => $hashtag->id,
], [
'related_tags' => array_values($filtered),
'agg_score' => $agg_score,
'last_calculated_at' => now()
]);
$this->info('Finished!');
}
}

View file

@ -0,0 +1,140 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Media;
use Cache, Storage;
use Illuminate\Contracts\Console\PromptsForMissingInput;
class MediaCloudUrlRewrite extends Command implements PromptsForMissingInput
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:cloud-url-rewrite {oldDomain} {newDomain}';
/**
* Prompt for missing input arguments using the returned questions.
*
* @return array
*/
protected function promptForMissingArgumentsUsing()
{
return [
'oldDomain' => 'The old S3 domain',
'newDomain' => 'The new S3 domain'
];
}
/**
* The console command description.
*
* @var string
*/
protected $description = 'Rewrite S3 media urls from local users';
/**
* Execute the console command.
*/
public function handle()
{
$this->preflightCheck();
$this->bootMessage();
$this->confirmCloudUrl();
}
protected function preflightCheck()
{
if(config_cache('pixelfed.cloud_storage') != true) {
$this->info('Error: Cloud storage is not enabled!');
$this->error('Aborting...');
exit;
}
}
protected function bootMessage()
{
$this->info(' ____ _ ______ __ ');
$this->info(' / __ \(_) _____ / / __/__ ____/ / ');
$this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / ');
$this->info(' / ____/ /> </ __/ / __/ __/ /_/ / ');
$this->info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ ');
$this->info(' ');
$this->info(' Media Cloud Url Rewrite Tool');
$this->info(' ===');
$this->info(' Old S3: ' . trim($this->argument('oldDomain')));
$this->info(' New S3: ' . trim($this->argument('newDomain')));
$this->info(' ');
}
protected function confirmCloudUrl()
{
$disk = Storage::disk(config('filesystems.cloud'))->url('test');
$domain = parse_url($disk, PHP_URL_HOST);
if(trim($this->argument('newDomain')) !== $domain) {
$this->error('Error: The new S3 domain you entered is not currently configured');
exit;
}
if(!$this->confirm('Confirm this is correct')) {
$this->error('Aborting...');
exit;
}
$this->updateUrls();
}
protected function updateUrls()
{
$this->info('Updating urls...');
$oldDomain = trim($this->argument('oldDomain'));
$newDomain = trim($this->argument('newDomain'));
$disk = Storage::disk(config('filesystems.cloud'));
$count = Media::whereNotNull('cdn_url')->count();
$bar = $this->output->createProgressBar($count);
$counter = 0;
$bar->start();
foreach(Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) {
if(strncmp($media->media_path, 'http', 4) === 0) {
$bar->advance();
continue;
}
$cdnHost = parse_url($media->cdn_url, PHP_URL_HOST);
if($oldDomain != $cdnHost || $newDomain == $cdnHost) {
$bar->advance();
continue;
}
$media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url);
if($media->thumbnail_url != null) {
$thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST);
if($thumbHost == $oldDomain) {
$thumbUrl = $disk->url($media->thumbnail_path);
$media->thumbnail_url = $thumbUrl;
}
}
if($media->optimized_url != null) {
$optiHost = parse_url($media->optimized_url, PHP_URL_HOST);
if($optiHost == $oldDomain) {
$optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url);
$media->optimized_url = $optiUrl;
}
}
$media->save();
$counter++;
$bar->advance();
}
$bar->finish();
$this->line(' ');
$this->info('Finished! Updated ' . $counter . ' total records!');
$this->line(' ');
$this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
}
}

View file

@ -0,0 +1,31 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
class NotificationEpochUpdate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:notification-epoch-update';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Update notification epoch';
/**
* Execute the console command.
*/
public function handle()
{
NotificationEpochUpdatePipeline::dispatch();
}
}

View file

@ -3,16 +3,17 @@
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Contracts\Console\PromptsForMissingInput;
use App\User;
class UserAdmin extends Command
class UserAdmin extends Command implements PromptsForMissingInput
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'user:admin {id}';
protected $signature = 'user:admin {username}';
/**
* The console command description.
@ -22,13 +23,15 @@ class UserAdmin extends Command
protected $description = 'Make a user an admin, or remove admin privileges.';
/**
* Create a new command instance.
* Prompt for missing input arguments using the returned questions.
*
* @return void
* @return array
*/
public function __construct()
protected function promptForMissingArgumentsUsing()
{
parent::__construct();
return [
'username' => 'Which username should we toggle admin privileges for?',
];
}
/**
@ -38,16 +41,15 @@ class UserAdmin extends Command
*/
public function handle()
{
$id = $this->argument('id');
if(ctype_digit($id) == true) {
$user = User::find($id);
} else {
$id = $this->argument('username');
$user = User::whereUsername($id)->first();
}
if(!$user) {
$this->error('Could not find any user with that username or id.');
exit;
}
$this->info('Found username: ' . $user->username);
$state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?';
$confirmed = $this->confirm($state);

View file

@ -0,0 +1,61 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Contracts\Console\PromptsForMissingInput;
use App\User;
class UserToggle2FA extends Command implements PromptsForMissingInput
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'user:2fa {username}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Disable two factor authentication for given username';
/**
* Prompt for missing input arguments using the returned questions.
*
* @return array
*/
protected function promptForMissingArgumentsUsing()
{
return [
'username' => 'Which username should we disable 2FA for?',
];
}
/**
* Execute the console command.
*/
public function handle()
{
$user = User::whereUsername($this->argument('username'))->first();
if(!$user) {
$this->error('Could not find any user with that username');
exit;
}
if(!$user->{'2fa_enabled'}) {
$this->info('User did not have 2FA enabled!');
return;
}
$user->{'2fa_enabled'} = false;
$user->{'2fa_secret'} = null;
$user->{'2fa_backup_codes'} = null;
$user->save();
$this->info('Successfully disabled 2FA on this account!');
}
}

View file

@ -43,6 +43,9 @@ class Kernel extends ConsoleKernel
$schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37)->onOneServer();
$schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32)->onOneServer();
}
$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);
}
/**

View file

@ -643,7 +643,7 @@ trait AdminReportController
$q->whereNull('admin_seen') :
$q->whereNotNull('admin_seen');
})
->groupBy(['object_id', 'object_type'])
->groupBy(['id', 'object_id', 'object_type', 'profile_id'])
->cursorPaginate(6)
->withQueryString()
);

View file

@ -0,0 +1,123 @@
<?php
namespace App\Http\Controllers;
use Illuminate\Http\Request;
use App\Models\AdminShadowFilter;
use App\Profile;
use App\Services\AccountService;
use App\Services\AdminShadowFilterService;
class AdminShadowFilterController extends Controller
{
public function __construct()
{
$this->middleware(['auth','admin']);
}
public function home(Request $request)
{
$filter = $request->input('filter');
$searchQuery = $request->input('q');
$filters = AdminShadowFilter::whereHas('profile')
->when($filter, function($q, $filter) {
if($filter == 'all') {
return $q;
} else if($filter == 'inactive') {
return $q->whereActive(false);
} else {
return $q;
}
}, function($q, $filter) {
return $q->whereActive(true);
})
->when($searchQuery, function($q, $searchQuery) {
$ids = Profile::where('username', 'like', '%' . $searchQuery . '%')
->limit(100)
->pluck('id')
->toArray();
return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids);
})
->latest()
->paginate(10)
->withQueryString();
return view('admin.asf.home', compact('filters'));
}
public function create(Request $request)
{
return view('admin.asf.create');
}
public function edit(Request $request, $id)
{
$filter = AdminShadowFilter::findOrFail($id);
$profile = AccountService::get($filter->item_id);
return view('admin.asf.edit', compact('filter', 'profile'));
}
public function store(Request $request)
{
$this->validate($request, [
'username' => 'required',
'active' => 'sometimes',
'note' => 'sometimes',
'hide_from_public_feeds' => 'sometimes'
]);
$profile = Profile::whereUsername($request->input('username'))->first();
if(!$profile) {
return back()->withErrors(['Invalid account']);
}
if($profile->user && $profile->user->is_admin) {
return back()->withErrors(['Cannot filter an admin account']);
}
$active = $request->has('active') && $request->has('hide_from_public_feeds');
AdminShadowFilter::updateOrCreate([
'item_id' => $profile->id,
'item_type' => get_class($profile)
], [
'is_local' => $profile->domain === null,
'note' => $request->input('note'),
'hide_from_public_feeds' => $request->has('hide_from_public_feeds'),
'admin_id' => $request->user()->profile_id,
'active' => $active
]);
AdminShadowFilterService::refresh();
return redirect('/i/admin/asf/home');
}
public function storeEdit(Request $request, $id)
{
$this->validate($request, [
'active' => 'sometimes',
'note' => 'sometimes',
'hide_from_public_feeds' => 'sometimes'
]);
$filter = AdminShadowFilter::findOrFail($id);
$profile = Profile::findOrFail($filter->item_id);
if($profile->user && $profile->user->is_admin) {
return back()->withErrors(['Cannot filter an admin account']);
}
$active = $request->has('active');
$filter->active = $active;
$filter->hide_from_public_feeds = $request->has('hide_from_public_feeds');
$filter->note = $request->input('note');
$filter->save();
AdminShadowFilterService::refresh();
return redirect('/i/admin/asf/home');
}
}

View file

@ -31,6 +31,7 @@ use App\{
UserSetting,
UserFilter,
};
use App\Models\UserDomainBlock;
use League\Fractal;
use App\Transformer\Api\Mastodon\v1\{
AccountTransformer,
@ -71,6 +72,8 @@ use App\Services\{
CollectionService,
FollowerService,
HashtagService,
HashtagFollowService,
HomeTimelineService,
InstanceService,
LikeService,
NetworkTimelineService,
@ -102,6 +105,8 @@ use Illuminate\Support\Facades\RateLimiter;
use Purify;
use Carbon\Carbon;
use App\Http\Resources\MastoApi\FollowedTagResource;
use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
class ApiV1Controller extends Controller
{
@ -185,6 +190,7 @@ class ApiV1Controller extends Controller
abort_if(!$user, 403);
abort_if($user->status != null, 403);
AccountService::setLastActive($user->id);
$res = $request->has(self::PF_API_ENTITY_KEY) ? AccountService::get($user->profile_id) : AccountService::getMastodon($user->profile_id);
@ -243,6 +249,7 @@ class ApiV1Controller extends Controller
]);
$user = $request->user();
AccountService::setLastActive($user->id);
$profile = $user->profile;
$settings = $user->settings;
@ -750,6 +757,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$target = Profile::where('id', '!=', $user->profile_id)
->whereNull('status')
@ -834,6 +842,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$target = Profile::where('id', '!=', $user->profile_id)
->whereNull('status')
@ -937,6 +946,7 @@ class ApiV1Controller extends Controller
]);
$user = $request->user();
AccountService::setLastActive($user->id);
$query = $request->input('q');
$limit = $request->input('limit') ?? 20;
$resolve = (bool) $request->input('resolve', false);
@ -1009,6 +1019,7 @@ class ApiV1Controller extends Controller
$user = $request->user();
$pid = $user->profile_id ?? $user->profile->id;
AccountService::setLastActive($user->id);
if(intval($id) === intval($pid)) {
abort(400, 'You cannot block yourself');
@ -1101,6 +1112,7 @@ class ApiV1Controller extends Controller
$user = $request->user();
$pid = $user->profile_id ?? $user->profile->id;
AccountService::setLastActive($user->id);
if(intval($id) === intval($pid)) {
abort(400, 'You cannot unblock yourself');
@ -1233,6 +1245,8 @@ class ApiV1Controller extends Controller
$user = $request->user();
AccountService::setLastActive($user->id);
$status = StatusService::getMastodon($id, false);
abort_unless($status, 400);
@ -1271,7 +1285,7 @@ class ApiV1Controller extends Controller
Status::findOrFail($status['id'])->update([
'likes_count' => ($status['favourites_count'] ?? 0) + 1
]);
LikePipeline::dispatch($like);
LikePipeline::dispatch($like)->onQueue('feed');
}
$status['favourited'] = true;
@ -1292,6 +1306,8 @@ class ApiV1Controller extends Controller
$user = $request->user();
AccountService::setLastActive($user->id);
$status = Status::findOrFail($id);
if(intval($status->profile_id) !== intval($user->profile_id)) {
@ -1308,7 +1324,7 @@ class ApiV1Controller extends Controller
if($like) {
$like->forceDelete();
$status->likes_count = $status->likes()->count();
$status->likes_count = $status->likes_count > 1 ? $status->likes_count - 1 : 0;
$status->save();
}
@ -1607,6 +1623,7 @@ class ApiV1Controller extends Controller
]);
$user = $request->user();
AccountService::setLastActive($user->id);
if($user->last_active_at == null) {
return [];
@ -1728,6 +1745,7 @@ class ApiV1Controller extends Controller
]);
$user = $request->user();
AccountService::setLastActive($user->id);
$media = Media::whereUserId($user->id)
->whereProfileId($user->profile_id)
@ -1774,6 +1792,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$media = Media::whereUserId($user->id)
->whereNull('status_id')
@ -1817,6 +1836,8 @@ class ApiV1Controller extends Controller
return [];
}
AccountService::setLastActive($user->id);
if(empty($request->file('file'))) {
return response('', 422);
}
@ -2061,6 +2082,8 @@ class ApiV1Controller extends Controller
'min_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
'max_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
'since_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX,
'types[]' => 'sometimes|array',
'type' => 'sometimes|string|in:mention,reblog,follow,favourite'
]);
$pid = $request->user()->profile_id;
@ -2074,29 +2097,29 @@ class ApiV1Controller extends Controller
$min = 1;
}
$types = $request->input('types');
$maxId = null;
$minId = null;
AccountService::setLastActive($request->user()->id);
if($max) {
$res = NotificationService::getMaxMastodon($pid, $max, $limit);
$ids = NotificationService::getRankedMaxId($pid, $max, $limit);
$res = $max ?
NotificationService::getMaxMastodon($pid, $max, $limit) :
NotificationService::getMinMastodon($pid, $min ?? $since, $limit);
$ids = $max ?
NotificationService::getRankedMaxId($pid, $max, $limit) :
NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
if(!empty($ids)) {
$maxId = max($ids);
$minId = min($ids);
}
} else {
$res = NotificationService::getMinMastodon($pid, $min ?? $since, $limit);
$ids = NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
if(!empty($ids)) {
$maxId = max($ids);
$minId = min($ids);
}
}
if(empty($res) && !Cache::has('pf:services:notifications:hasSynced:'.$pid)) {
if(empty($res)) {
if(!Cache::has('pf:services:notifications:hasSynced:'.$pid)) {
Cache::put('pf:services:notifications:hasSynced:'.$pid, 1, 1209600);
NotificationService::warmCache($pid, 400, true);
}
}
$baseUrl = config('app.url') . '/api/v1/notifications?limit=' . $limit . '&';
@ -2132,7 +2155,7 @@ class ApiV1Controller extends Controller
'page' => 'sometimes|integer|max:40',
'min_id' => 'sometimes|integer|min:0|max:' . PHP_INT_MAX,
'max_id' => 'sometimes|integer|min:0|max:' . PHP_INT_MAX,
'limit' => 'sometimes|integer|min:1|max:100',
'limit' => 'sometimes|integer|min:1|max:40',
'include_reblogs' => 'sometimes',
]);
@ -2142,19 +2165,100 @@ class ApiV1Controller extends Controller
$max = $request->input('max_id');
$limit = $request->input('limit') ?? 20;
$pid = $request->user()->profile_id;
$includeReblogs = $request->filled('include_reblogs');
$includeReblogs = $request->filled('include_reblogs') ? $request->boolean('include_reblogs') : false;
$nullFields = $includeReblogs ?
['in_reply_to_id'] :
['in_reply_to_id', 'reblog_of_id'];
$inTypes = $includeReblogs ?
['photo', 'photo:album', 'video', 'video:album', 'photo:video:album', 'share'] :
['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'];
AccountService::setLastActive($request->user()->id);
if(config('exp.cached_home_timeline')) {
$paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50;
if($min || $max) {
if($request->has('min_id')) {
$res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
} else {
$res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
}
} else {
$res = HomeTimelineService::get($pid, 0, $paddedLimit);
}
if(!$res) {
$res = Cache::has('pf:services:apiv1:home:cached:coldbootcheck:' . $pid);
if(!$res) {
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:' . $pid, 1, 86400);
FeedWarmCachePipeline::dispatchSync($pid);
return response()->json([], 206);
} else {
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:' . $pid, 1, 86400);
return response()->json([], 206);
}
}
$res = collect($res)
->map(function($id) use($napi) {
return $napi ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
})
->filter(function($res) {
return $res && isset($res['account']);
})
->filter(function($s) use($includeReblogs) {
return $includeReblogs ? true : $s['reblog'] == null;
})
->take($limit)
->map(function($status) use($pid) {
if($pid) {
$status['favourited'] = (bool) LikeService::liked($pid, $status['id']);
$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
return $status;
})
->values();
$baseUrl = config('app.url') . '/api/v1/timelines/home?limit=' . $limit . '&';
$minId = $res->map(function($s) {
return ['id' => $s['id']];
})->min('id');
$maxId = $res->map(function($s) {
return ['id' => $s['id']];
})->max('id');
if($minId == $maxId) {
$minId = null;
}
if($maxId) {
$link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next"';
}
if($minId) {
$link = '<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
}
if($maxId && $minId) {
$link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next",<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
}
$headers = isset($link) ? ['Link' => $link] : [];
return $this->json($res->toArray(), 200, $headers);
}
$following = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
$following = Follower::whereProfileId($pid)->pluck('following_id');
return $following->push($pid)->toArray();
});
$muted = UserFilterService::mutes($pid);
if($muted && count($muted)) {
$following = array_diff($following, $muted);
}
if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
@ -2315,10 +2419,13 @@ class ApiV1Controller extends Controller
$max = $request->input('max_id');
$limit = $request->input('limit') ?? 20;
$user = $request->user();
$remote = ($request->has('remote') && $request->input('remote') == true) || ($request->filled('local') && $request->input('local') != true);
$remote = $request->has('remote');
$local = $request->has('local');
$filtered = $user ? UserFilterService::filters($user->profile_id) : [];
AccountService::setLastActive($user->id);
$domainBlocks = UserFilterService::domainBlocks($user->profile_id);
if((!$request->has('local') || $remote) && config('instance.timeline.network.cached')) {
if($remote && config('instance.timeline.network.cached')) {
Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
if(NetworkTimelineService::count() == 0) {
NetworkTimelineService::warmCache(true, config('instance.timeline.network.cache_dropoff'));
@ -2332,7 +2439,9 @@ class ApiV1Controller extends Controller
} else {
$feed = NetworkTimelineService::get(0, $limit + 5);
}
} else {
}
if($local || !$remote && !$local) {
Cache::remember('api:v1:timelines:public:cache_check', 10368000, function() {
if(PublicTimelineService::count() == 0) {
PublicTimelineService::warmCache(true, 400);
@ -2389,6 +2498,13 @@ class ApiV1Controller extends Controller
->filter(function($s) use($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->filter(function($s) use($domainBlocks) {
if(!$domainBlocks || !count($domainBlocks)) {
return $s;
}
$domain = strtolower(parse_url($s['url'], PHP_URL_HOST));
return !in_array($domain, $domainBlocks);
})
->take($limit)
->values();
@ -2506,8 +2622,8 @@ class ApiV1Controller extends Controller
public function statusById(Request $request, $id)
{
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($request->user()->id);
$pid = $request->user()->profile_id;
$res = $request->has(self::PF_API_ENTITY_KEY) ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
if(!$res || !isset($res['visibility'])) {
@ -2517,17 +2633,23 @@ class ApiV1Controller extends Controller
$scope = $res['visibility'];
if(!in_array($scope, ['public', 'unlisted'])) {
if($scope === 'private') {
if(intval($res['account']['id']) !== intval($user->profile_id)) {
abort_unless(FollowerService::follows($user->profile_id, $res['account']['id']), 403);
if(intval($res['account']['id']) !== intval($pid)) {
abort_unless(FollowerService::follows($pid, $res['account']['id']), 403);
}
} else {
abort(400, 'Invalid request');
}
}
$res['favourited'] = LikeService::liked($user->profile_id, $res['id']);
$res['reblogged'] = ReblogService::get($user->profile_id, $res['id']);
$res['bookmarked'] = BookmarkService::get($user->profile_id, $res['id']);
if(!empty($res['reblog']) && isset($res['reblog']['id'])) {
$res['reblog']['favourited'] = (bool) LikeService::liked($pid, $res['reblog']['id']);
$res['reblog']['reblogged'] = (bool) ReblogService::get($pid, $res['reblog']['id']);
$res['reblog']['bookmarked'] = BookmarkService::get($pid, $res['reblog']['id']);
}
$res['favourited'] = LikeService::liked($pid, $res['id']);
$res['reblogged'] = ReblogService::get($pid, $res['id']);
$res['bookmarked'] = BookmarkService::get($pid, $res['id']);
return $this->json($res);
}
@ -2544,6 +2666,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$pid = $user->profile_id;
$status = StatusService::getMastodon($id, false);
@ -3014,7 +3137,7 @@ class ApiV1Controller extends Controller
public function statusDelete(Request $request, $id)
{
abort_if(!$request->user(), 403);
AccountService::setLastActive($request->user()->id);
$status = Status::whereProfileId($request->user()->profile->id)
->findOrFail($id);
@ -3042,6 +3165,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$status = Status::whereScope('public')->findOrFail($id);
if(intval($status->profile_id) !== intval($user->profile_id)) {
@ -3088,6 +3212,7 @@ class ApiV1Controller extends Controller
abort_if(!$request->user(), 403);
$user = $request->user();
AccountService::setLastActive($user->id);
$status = Status::whereScope('public')->findOrFail($id);
if(intval($status->profile_id) !== intval($user->profile_id)) {
@ -3160,6 +3285,7 @@ class ApiV1Controller extends Controller
$limit = $request->input('limit', 20);
$onlyMedia = $request->input('only_media', true);
$pe = $request->has(self::PF_API_ENTITY_KEY);
$pid = $request->user()->profile_id;
if($min || $max) {
$minMax = SnowflakeService::byDate(now()->subMonths(6));
@ -3171,7 +3297,8 @@ class ApiV1Controller extends Controller
}
}
$filters = UserFilterService::filters($request->user()->profile_id);
$filters = UserFilterService::filters($pid);
$domainBlocks = UserFilterService::domainBlocks($pid);
if(!$min && !$max) {
$id = 1;
@ -3197,10 +3324,11 @@ class ApiV1Controller extends Controller
if($onlyMedia && !isset($i['media_attachments']) || !count($i['media_attachments'])) {
return false;
}
return $i && isset($i['account']);
return $i && isset($i['account'], $i['url']);
})
->filter(function($i) use($filters) {
return !in_array($i['account']['id'], $filters);
->filter(function($i) use($filters, $domainBlocks) {
$domain = strtolower(parse_url($i['url'], PHP_URL_HOST));
return !in_array($i['account']['id'], $filters) && !in_array($domain, $domainBlocks);
})
->values()
->toArray();
@ -3503,38 +3631,31 @@ class ApiV1Controller extends Controller
$pid = $request->user()->profile_id;
$ids = Cache::remember('api:v1.1:discover:accounts:popular', 86400, function() {
$ids = Cache::remember('api:v1.1:discover:accounts:popular', 3600, function() {
return DB::table('profiles')
->where('is_private', false)
->whereNull('status')
->orderByDesc('profiles.followers_count')
->limit(20)
->limit(30)
->get();
});
$filters = UserFilterService::filters($pid);
$ids = $ids->map(function($profile) {
return AccountService::get($profile->id, true);
})
->filter(function($profile) use($pid) {
return $profile && isset($profile['id']);
return $profile && isset($profile['id'], $profile['locked']) && !$profile['locked'];
})
->filter(function($profile) use($pid) {
return $profile['id'] != $pid;
})
->map(function($profile) {
$ids = collect(ProfileStatusService::get($profile['id'], 0, 9))
->map(function($id) {
return StatusService::get($id, true);
->filter(function($profile) use($pid) {
return !FollowerService::follows($pid, $profile['id'], true);
})
->filter(function($post) {
return $post && isset($post['id']);
->filter(function($profile) use($filters) {
return !in_array($profile['id'], $filters);
})
->take(3)
->values();
$profile['recent_posts'] = $ids;
return $profile;
})
->take(6)
->take(16)
->values();
return $this->json($ids);
@ -3633,168 +3754,4 @@ class ApiV1Controller extends Controller
return $this->json([]);
}
/**
* GET /api/v1/followed_tags
*
*
* @return array
*/
public function getFollowedTags(Request $request)
{
abort_if(!$request->user(), 403);
$account = AccountService::get($request->user()->profile_id);
$this->validate($request, [
'cursor' => 'sometimes',
'limit' => 'sometimes|integer|min:1|max:200'
]);
$limit = $request->input('limit', 100);
$res = HashtagFollow::whereProfileId($account['id'])
->orderByDesc('id')
->cursorPaginate($limit)->withQueryString();
$pagination = false;
$prevPage = $res->nextPageUrl();
$nextPage = $res->previousPageUrl();
if($nextPage && $prevPage) {
$pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
} else if($nextPage && !$prevPage) {
$pagination = '<' . $nextPage . '>; rel="next"';
} else if(!$nextPage && $prevPage) {
$pagination = '<' . $prevPage . '>; rel="prev"';
}
if($pagination) {
return response()->json(FollowedTagResource::collection($res)->collection)
->header('Link', $pagination);
}
return response()->json(FollowedTagResource::collection($res)->collection);
}
/**
* POST /api/v1/tags/:id/follow
*
*
* @return object
*/
public function followHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
abort_if(
HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
422,
'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
);
$follows = HashtagFollow::updateOrCreate(
[
'profile_id' => $account['id'],
'hashtag_id' => $tag->id
],
[
'user_id' => $request->user()->id
]
);
HashtagService::follow($pid, $tag->id);
return response()->json(FollowedTagResource::make($follows)->toArray($request));
}
/**
* POST /api/v1/tags/:id/unfollow
*
*
* @return object
*/
public function unfollowHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
$follows = HashtagFollow::whereProfileId($pid)
->whereHashtagId($tag->id)
->first();
if(!$follows) {
return [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => false
];
}
if($follows) {
HashtagService::unfollow($pid, $tag->id);
$follows->delete();
}
$res = FollowedTagResource::make($follows)->toArray($request);
$res['following'] = false;
return response()->json($res);
}
/**
* GET /api/v1/tags/:id
*
*
* @return object
*/
public function getHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
if(!$tag) {
return [
'name' => $id,
'url' => config('app.url') . '/i/web/hashtag/' . $id,
'history' => [],
'following' => false
];
}
$res = [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => HashtagService::isFollowing($pid, $tag->id)
];
if($request->has(self::PF_API_ENTITY_KEY)) {
$res['count'] = HashtagService::count($tag->id);
}
return $this->json($res);
}
}

View file

@ -11,6 +11,7 @@ use League\Fractal\Serializer\ArraySerializer;
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
use App\AccountLog;
use App\EmailVerification;
use App\Follower;
use App\Place;
use App\Status;
use App\Report;
@ -19,8 +20,11 @@ use App\StatusArchived;
use App\User;
use App\UserSetting;
use App\Services\AccountService;
use App\Services\FollowerService;
use App\Services\StatusService;
use App\Services\ProfileStatusService;
use App\Services\LikeService;
use App\Services\ReblogService;
use App\Services\PublicTimelineService;
use App\Services\NetworkTimelineService;
use App\Util\Lexer\RestrictedNames;
@ -470,7 +474,7 @@ class ApiV1Dot1Controller extends Controller
abort_if(BouncerService::checkIp($request->ip()), 404);
}
$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), 3, function(){}, 1800);
$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), config('pixelfed.app_registration_rate_limit_attempts', 3), function(){}, config('pixelfed.app_registration_rate_limit_decay', 1800));
abort_if(!$rl, 400, 'Too many requests');
$this->validate($request, [
@ -543,10 +547,10 @@ class ApiV1Dot1Controller extends Controller
$user->password = Hash::make($password);
$user->register_source = 'app';
$user->app_register_ip = $request->ip();
$user->app_register_token = Str::random(32);
$user->app_register_token = Str::random(40);
$user->save();
$rtoken = Str::random(mt_rand(64, 70));
$rtoken = Str::random(64);
$verify = new EmailVerification();
$verify->user_id = $user->id;
@ -555,7 +559,12 @@ class ApiV1Dot1Controller extends Controller
$verify->random_token = $rtoken;
$verify->save();
$appUrl = url('/api/v1.1/auth/iarer?ut=' . $user->app_register_token . '&rt=' . $rtoken);
$params = http_build_query([
'ut' => $user->app_register_token,
'rt' => $rtoken,
'ea' => base64_encode($user->email)
]);
$appUrl = url('/api/v1.1/auth/iarer?'. $params);
Mail::to($user->email)->send(new ConfirmAppEmail($verify, $appUrl));
@ -568,14 +577,19 @@ class ApiV1Dot1Controller extends Controller
{
$this->validate($request, [
'ut' => 'required',
'rt' => 'required'
'rt' => 'required',
'ea' => 'required'
]);
if(config('pixelfed.bouncer.cloud_ips.ban_signups')) {
abort_if(BouncerService::checkIp($request->ip()), 404);
}
$ut = $request->input('ut');
$rt = $request->input('rt');
$url = 'pixelfed://confirm-account/'. $ut . '?rt=' . $rt;
$ea = $request->input('ea');
$params = http_build_query([
'ut' => $ut,
'rt' => $rt,
'domain' => config('pixelfed.domain.app'),
'ea' => $ea
]);
$url = 'pixelfed://confirm-account/'. $ut . '?' . $params;
return redirect()->away($url);
}
@ -589,8 +603,8 @@ class ApiV1Dot1Controller extends Controller
abort_if(BouncerService::checkIp($request->ip()), 404);
}
$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), 10, function(){}, 1800);
abort_if(!$rl, 400, 'Too many requests');
$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), config('pixelfed.app_registration_confirm_rate_limit_attempts', 20), function(){}, config('pixelfed.app_registration_confirm_rate_limit_decay', 1800));
abort_if(!$rl, 429, 'Too many requests');
$this->validate($request, [
'user_token' => 'required',
@ -884,4 +898,19 @@ class ApiV1Dot1Controller extends Controller
return [200];
}
public function getMutualAccounts(Request $request, $id)
{
abort_if(!$request->user(), 403);
$account = AccountService::get($id, true);
if(!$account || !isset($account['id'])) { return []; }
$res = collect(FollowerService::mutualAccounts($request->user()->profile_id, $id))
->map(function($accountId) {
return AccountService::get($accountId, true);
})
->filter()
->take(24)
->values();
return $this->json($res);
}
}

View file

@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{
use App\Transformer\Api\{
RelationshipTransformer,
};
use App\Util\Site\Nodeinfo;
class ApiV2Controller extends Controller
{
@ -77,12 +78,7 @@ class ApiV2Controller extends Controller
'description' => config_cache('app.short_description'),
'usage' => [
'users' => [
'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() {
return User::select('last_active_at', 'created_at')
->where('last_active_at', '>', now()->subMonths(1))
->orWhere('created_at', '>', now()->subMonths(1))
->count();
})
'active_month' => (int) Nodeinfo::activeUsersMonthly()
]
],
'thumbnail' => [

View file

@ -0,0 +1,118 @@
<?php
namespace App\Http\Controllers\Api\V1;
use Illuminate\Http\Request;
use App\Http\Controllers\Controller;
use App\Models\UserDomainBlock;
use App\Util\ActivityPub\Helpers;
use App\Services\UserFilterService;
use Illuminate\Bus\Batch;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Cache;
use App\Jobs\HomeFeedPipeline\FeedRemoveDomainPipeline;
use App\Jobs\ProfilePipeline\ProfilePurgeNotificationsByDomain;
use App\Jobs\ProfilePipeline\ProfilePurgeFollowersByDomain;
class DomainBlockController extends Controller
{
public function json($res, $code = 200, $headers = [])
{
return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
}
public function index(Request $request)
{
abort_unless($request->user(), 403);
$this->validate($request, [
'limit' => 'sometimes|integer|min:1|max:200'
]);
$limit = $request->input('limit', 100);
$id = $request->user()->profile_id;
$filters = UserDomainBlock::whereProfileId($id)->orderByDesc('id')->cursorPaginate($limit);
$links = null;
$headers = [];
if($filters->nextCursor()) {
$links .= '<'.$filters->nextPageUrl().'&limit='.$limit.'>; rel="next"';
}
if($filters->previousCursor()) {
if($links != null) {
$links .= ', ';
}
$links .= '<'.$filters->previousPageUrl().'&limit='.$limit.'>; rel="prev"';
}
if($links) {
$headers = ['Link' => $links];
}
return $this->json($filters->pluck('domain'), 200, $headers);
}
public function store(Request $request)
{
abort_unless($request->user(), 403);
$this->validate($request, [
'domain' => 'required|active_url|min:1|max:120'
]);
$pid = $request->user()->profile_id;
$domain = trim($request->input('domain'));
if(Helpers::validateUrl($domain) == false) {
return abort(500, 'Invalid domain or already blocked by server admins');
}
$domain = strtolower(parse_url($domain, PHP_URL_HOST));
abort_if(config_cache('pixelfed.domain.app') == $domain, 400, 'Cannot ban your own server');
$existingCount = UserDomainBlock::whereProfileId($pid)->count();
$maxLimit = config('instance.user_filters.max_domain_blocks');
$errorMsg = __('profile.block.domain.max', ['max' => $maxLimit]);
abort_if($existingCount >= $maxLimit, 400, $errorMsg);
$block = UserDomainBlock::updateOrCreate([
'profile_id' => $pid,
'domain' => $domain
]);
if($block->wasRecentlyCreated) {
Bus::batch([
[
new FeedRemoveDomainPipeline($pid, $domain),
new ProfilePurgeNotificationsByDomain($pid, $domain),
new ProfilePurgeFollowersByDomain($pid, $domain)
]
])->allowFailures()->onQueue('feed')->dispatch();
Cache::forget('profile:following:' . $pid);
UserFilterService::domainBlocks($pid, true);
}
return $this->json([]);
}
public function delete(Request $request)
{
abort_unless($request->user(), 403);
$this->validate($request, [
'domain' => 'required|min:1|max:120'
]);
$pid = $request->user()->profile_id;
$domain = strtolower(trim($request->input('domain')));
$filters = UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->delete();
UserFilterService::domainBlocks($pid, true);
return $this->json([]);
}
}

View file

@ -0,0 +1,207 @@
<?php
namespace App\Http\Controllers\Api\V1;
use Illuminate\Http\Request;
use App\Http\Controllers\Controller;
use App\Hashtag;
use App\HashtagFollow;
use App\StatusHashtag;
use App\Services\AccountService;
use App\Services\HashtagService;
use App\Services\HashtagFollowService;
use App\Services\HashtagRelatedService;
use App\Http\Resources\MastoApi\FollowedTagResource;
use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
class TagsController extends Controller
{
const PF_API_ENTITY_KEY = "_pe";
public function json($res, $code = 200, $headers = [])
{
return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
}
/**
* GET /api/v1/tags/:id/related
*
*
* @return array
*/
public function relatedTags(Request $request, $tag)
{
abort_unless($request->user(), 403);
$tag = Hashtag::whereSlug($tag)->firstOrFail();
return HashtagRelatedService::get($tag->id);
}
/**
* POST /api/v1/tags/:id/follow
*
*
* @return object
*/
public function followHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
abort_if(
HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
422,
'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
);
$follows = HashtagFollow::updateOrCreate(
[
'profile_id' => $account['id'],
'hashtag_id' => $tag->id
],
[
'user_id' => $request->user()->id
]
);
HashtagService::follow($pid, $tag->id);
HashtagFollowService::add($tag->id, $pid);
return response()->json(FollowedTagResource::make($follows)->toArray($request));
}
/**
* POST /api/v1/tags/:id/unfollow
*
*
* @return object
*/
public function unfollowHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
$follows = HashtagFollow::whereProfileId($pid)
->whereHashtagId($tag->id)
->first();
if(!$follows) {
return [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => false
];
}
if($follows) {
HashtagService::unfollow($pid, $tag->id);
HashtagFollowService::unfollow($tag->id, $pid);
HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
$follows->delete();
}
$res = FollowedTagResource::make($follows)->toArray($request);
$res['following'] = false;
return response()->json($res);
}
/**
* GET /api/v1/tags/:id
*
*
* @return object
*/
public function getHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
if(!$tag) {
return [
'name' => $id,
'url' => config('app.url') . '/i/web/hashtag/' . $id,
'history' => [],
'following' => false
];
}
$res = [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => HashtagService::isFollowing($pid, $tag->id)
];
if($request->has(self::PF_API_ENTITY_KEY)) {
$res['count'] = HashtagService::count($tag->id);
}
return $this->json($res);
}
/**
* GET /api/v1/followed_tags
*
*
* @return array
*/
public function getFollowedTags(Request $request)
{
abort_if(!$request->user(), 403);
$account = AccountService::get($request->user()->profile_id);
$this->validate($request, [
'cursor' => 'sometimes',
'limit' => 'sometimes|integer|min:1|max:200'
]);
$limit = $request->input('limit', 100);
$res = HashtagFollow::whereProfileId($account['id'])
->orderByDesc('id')
->cursorPaginate($limit)
->withQueryString();
$pagination = false;
$prevPage = $res->nextPageUrl();
$nextPage = $res->previousPageUrl();
if($nextPage && $prevPage) {
$pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
} else if($nextPage && !$prevPage) {
$pagination = '<' . $nextPage . '>; rel="next"';
} else if(!$nextPage && $prevPage) {
$pagination = '<' . $prevPage . '>; rel="prev"';
}
if($pagination) {
return response()->json(FollowedTagResource::collection($res)->collection)
->header('Link', $pagination);
}
return response()->json(FollowedTagResource::collection($res)->collection);
}
}

View file

@ -153,7 +153,7 @@ class CollectionController extends Controller
abort(400, 'You can only add '.$max.' posts per collection');
}
$status = Status::whereScope('public')
$status = Status::whereIn('scope', ['public', 'unlisted'])
->whereProfileId($profileId)
->whereIn('type', ['photo', 'photo:album', 'video'])
->findOrFail($postId);
@ -166,17 +166,13 @@ class CollectionController extends Controller
'order' => $count,
]);
CollectionService::addItem(
$collection->id,
$status->id,
$count
);
CollectionService::deleteCollection($collection->id);
$collection->updated_at = now();
$collection->save();
CollectionService::setCollection($collection->id, $collection);
return StatusService::get($status->id);
return StatusService::get($status->id, false);
}
public function getCollection(Request $request, $id)
@ -226,10 +222,10 @@ class CollectionController extends Controller
return collect($items)
->map(function($id) {
return StatusService::get($id);
return StatusService::get($id, false);
})
->filter(function($item) {
return $item && isset($item['account'], $item['media_attachments']);
return $item && ($item['visibility'] == 'public' || $item['visibility'] == 'unlisted') && isset($item['account'], $item['media_attachments']);
})
->values();
}
@ -298,7 +294,7 @@ class CollectionController extends Controller
abort(400, 'You cannot delete the only post of a collection!');
}
$status = Status::whereScope('public')
$status = Status::whereIn('scope', ['public', 'unlisted'])
->whereIn('type', ['photo', 'photo:album', 'video'])
->findOrFail($postId);

View file

@ -415,7 +415,7 @@ class ComposeController extends Controller
$results = Profile::select('id','domain','username')
->whereNotIn('id', $blocked)
->where('username','like','%'.$q.'%')
->groupBy('domain')
->groupBy('id', 'domain')
->limit(15)
->get()
->map(function($profile) {

View file

@ -17,12 +17,15 @@ use App\{
use App\Services\MediaPathService;
use App\Services\MediaBlocklistService;
use App\Jobs\StatusPipeline\NewStatusPipeline;
use App\Jobs\StatusPipeline\StatusDelete;
use Illuminate\Support\Str;
use App\Util\ActivityPub\Helpers;
use App\Services\AccountService;
use App\Services\StatusService;
use App\Services\WebfingerService;
use App\Models\Conversation;
use App\Jobs\DirectPipeline\DirectDeletePipeline;
use App\Jobs\DirectPipeline\DirectDeliverPipeline;
class DirectMessageController extends Controller
{
@ -500,6 +503,8 @@ class DirectMessageController extends Controller
if($recipient['local'] == false) {
$dmc = $dm;
$this->remoteDelete($dmc);
} else {
StatusDelete::dispatch($status)->onQueue('high');
}
if(Conversation::whereStatusId($sid)->count()) {
@ -541,9 +546,7 @@ class DirectMessageController extends Controller
StatusService::del($status->id, true);
$status->delete();
$dm->delete();
$status->forceDeleteQuietly();
return [200];
}
@ -829,7 +832,7 @@ class DirectMessageController extends Controller
]
];
Helpers::sendSignedObject($profile, $url, $body);
DirectDeliverPipeline::dispatch($profile, $url, $body)->onQueue('high');
}
public function remoteDelete($dm)
@ -852,7 +855,6 @@ class DirectMessageController extends Controller
'type' => 'Tombstone'
]
];
Helpers::sendSignedObject($profile, $url, $body);
DirectDeletePipeline::dispatch($profile, $url, $body)->onQueue('high');
}
}

View file

@ -30,6 +30,7 @@ use App\Util\ActivityPub\{
};
use Zttp\Zttp;
use App\Services\InstanceService;
use App\Services\AccountService;
class FederationController extends Controller
{
@ -123,12 +124,15 @@ class FederationController extends Controller
return redirect('/' . $username);
}
$id = AccountService::usernameToId($username);
abort_if(!$id, 404);
$account = AccountService::get($id);
abort_if(!$account || !isset($account['statuses_count']), 404);
$res = [
'@context' => 'https://www.w3.org/ns/activitystreams',
'id' => 'https://' . config('pixelfed.domain.app') . '/users/' . $username . '/outbox',
'type' => 'OrderedCollection',
'totalItems' => 0,
'orderedItems' => []
'totalItems' => $account['statuses_count'] ?? 0,
];
return response(json_encode($res, JSON_UNESCAPED_SLASHES))->header('Content-Type', 'application/activity+json');
@ -239,12 +243,15 @@ class FederationController extends Controller
{
abort_if(!config_cache('federation.activitypub.enabled'), 404);
$id = AccountService::usernameToId($username);
abort_if(!$id, 404);
$account = AccountService::get($id);
abort_if(!$account || !isset($account['following_count']), 404);
$obj = [
'@context' => 'https://www.w3.org/ns/activitystreams',
'id' => $request->getUri(),
'type' => 'OrderedCollectionPage',
'totalItems' => 0,
'orderedItems' => []
'type' => 'OrderedCollection',
'totalItems' => $account['following_count'] ?? 0,
];
return response()->json($obj);
}
@ -252,15 +259,16 @@ class FederationController extends Controller
public function userFollowers(Request $request, $username)
{
abort_if(!config_cache('federation.activitypub.enabled'), 404);
$id = AccountService::usernameToId($username);
abort_if(!$id, 404);
$account = AccountService::get($id);
abort_if(!$account || !isset($account['followers_count']), 404);
$obj = [
'@context' => 'https://www.w3.org/ns/activitystreams',
'id' => $request->getUri(),
'type' => 'OrderedCollectionPage',
'totalItems' => 0,
'orderedItems' => []
'type' => 'OrderedCollection',
'totalItems' => $account['followers_count'] ?? 0,
];
return response()->json($obj);
}
}

View file

@ -83,6 +83,17 @@ class ImportPostController extends Controller
);
}
public function formatHashtags($val = false)
{
if(!$val || !strlen($val)) {
return null;
}
$groupedHashtagRegex = '/#\w+(?=#)/';
return preg_replace($groupedHashtagRegex, '$0 ', $val);
}
public function store(Request $request)
{
abort_unless(config('import.instagram.enabled'), 404);
@ -128,11 +139,11 @@ class ImportPostController extends Controller
$ip->media = $c->map(function($m) {
return [
'uri' => $m['uri'],
'title' => $m['title'],
'title' => $this->formatHashtags($m['title']),
'creation_timestamp' => $m['creation_timestamp']
];
})->toArray();
$ip->caption = $c->count() > 1 ? $file['title'] : $ip->media[0]['title'];
$ip->caption = $c->count() > 1 ? $this->formatHashtags($file['title']) : $this->formatHashtags($ip->media[0]['title']);
$ip->filename = last(explode('/', $ip->media[0]['uri']));
$ip->metadata = $c->map(function($m) {
return [

View file

@ -25,8 +25,7 @@ class LikeController extends Controller
'item' => 'required|integer|min:1',
]);
// API deprecated
return;
abort(422, 'Deprecated API Endpoint');
$user = Auth::user();
$profile = $user->profile;
@ -34,7 +33,7 @@ class LikeController extends Controller
if (Like::whereStatusId($status->id)->whereProfileId($profile->id)->exists()) {
$like = Like::whereProfileId($profile->id)->whereStatusId($status->id)->firstOrFail();
UnlikePipeline::dispatch($like);
UnlikePipeline::dispatch($like)->onQueue('feed');
} else {
abort_if(
Like::whereProfileId($user->profile_id)
@ -60,7 +59,7 @@ class LikeController extends Controller
]) == false;
$like->save();
$status->save();
LikePipeline::dispatch($like);
LikePipeline::dispatch($like)->onQueue('feed');
}
}

View file

@ -23,7 +23,13 @@ class RemoteAuthController extends Controller
{
public function start(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
if($request->user()) {
return redirect('/');
}
@ -37,7 +43,13 @@ class RemoteAuthController extends Controller
public function getAuthDomains(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
if(config('remote-auth.mastodon.domains.only_custom')) {
$res = config('remote-auth.mastodon.domains.custom');
@ -69,7 +81,14 @@ class RemoteAuthController extends Controller
public function redirect(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
$this->validate($request, ['domain' => 'required']);
$domain = $request->input('domain');
@ -158,6 +177,14 @@ class RemoteAuthController extends Controller
public function preflight(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) {
return redirect('/login');
}
@ -167,6 +194,14 @@ class RemoteAuthController extends Controller
public function handleCallback(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
$domain = $request->session()->get('oauth_domain');
if($request->filled('code')) {
@ -195,7 +230,13 @@ class RemoteAuthController extends Controller
public function onboarding(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
if($request->user()) {
return redirect('/');
}
@ -204,6 +245,13 @@ class RemoteAuthController extends Controller
public function sessionCheck(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 403);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -248,6 +296,13 @@ class RemoteAuthController extends Controller
public function sessionGetMastodonData(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 403);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -279,6 +334,13 @@ class RemoteAuthController extends Controller
public function sessionValidateUsername(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 403);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -334,6 +396,13 @@ class RemoteAuthController extends Controller
public function sessionValidateEmail(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 403);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -359,6 +428,13 @@ class RemoteAuthController extends Controller
public function sessionGetMastodonFollowers(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@ -386,6 +462,13 @@ class RemoteAuthController extends Controller
public function handleSubmit(Request $request)
{
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
abort_unless($request->session()->exists('oauth_remasto_id'), 403);
@ -464,7 +547,13 @@ class RemoteAuthController extends Controller
public function storeBio(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_unless($request->user(), 404);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -483,7 +572,13 @@ class RemoteAuthController extends Controller
public function accountToId(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 404);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);
@ -525,7 +620,13 @@ class RemoteAuthController extends Controller
public function storeAvatar(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_unless($request->user(), 404);
$this->validate($request, [
'avatar_url' => 'required|active_url',
@ -547,7 +648,13 @@ class RemoteAuthController extends Controller
public function finishUp(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_unless($request->user(), 404);
$currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app');
@ -564,7 +671,13 @@ class RemoteAuthController extends Controller
public function handleLogin(Request $request)
{
abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404);
abort_unless((
config_cache('pixelfed.open_registration') &&
config('remote-auth.mastodon.enabled')
) || (
config('remote-auth.mastodon.ignore_closed_state') &&
config('remote-auth.mastodon.enabled')
), 404);
abort_if($request->user(), 404);
abort_unless($request->session()->exists('oauth_domain'), 403);
abort_unless($request->session()->exists('oauth_remote_session_token'), 403);

View file

@ -14,6 +14,7 @@ use App\Util\Lexer\PrettyNumber;
use App\Util\ActivityPub\Helpers;
use Auth, Cache, DB;
use Illuminate\Http\Request;
use App\Models\UserDomainBlock;
trait PrivacySettings
{
@ -39,9 +40,11 @@ trait PrivacySettings
'public_dm',
'show_profile_follower_count',
'show_profile_following_count',
'indexable',
'show_atom',
];
$profile->indexable = $request->input('indexable') == 'on';
$profile->is_suggestable = $request->input('is_suggestable') == 'on';
$profile->save();
@ -70,6 +73,8 @@ trait PrivacySettings
} else {
$settings->{$field} = false;
}
} elseif ($field == 'indexable') {
} else {
if ($form == 'on') {
$settings->{$field} = true;
@ -145,47 +150,25 @@ trait PrivacySettings
public function blockedInstances()
{
$pid = Auth::user()->profile->id;
$filters = UserFilter::whereUserId($pid)
->whereFilterableType('App\Instance')
->whereFilterType('block')
->orderByDesc('id')
->paginate(10);
return view('settings.privacy.blocked-instances', compact('filters'));
// deprecated
abort(404);
}
public function domainBlocks()
{
return view('settings.privacy.domain-blocks');
}
public function blockedInstanceStore(Request $request)
{
$this->validate($request, [
'domain' => 'required|url|min:1|max:120'
]);
$domain = $request->input('domain');
if(Helpers::validateUrl($domain) == false) {
return abort(400, 'Invalid domain');
}
$domain = parse_url($domain, PHP_URL_HOST);
$instance = Instance::firstOrCreate(['domain' => $domain]);
$filter = new UserFilter;
$filter->user_id = Auth::user()->profile->id;
$filter->filterable_id = $instance->id;
$filter->filterable_type = 'App\Instance';
$filter->filter_type = 'block';
$filter->save();
return response()->json(['msg' => 200]);
// deprecated
abort(404);
}
public function blockedInstanceUnblock(Request $request)
{
$this->validate($request, [
'id' => 'required|integer|min:1'
]);
$pid = Auth::user()->profile->id;
$filter = UserFilter::whereFilterableType('App\Instance')
->whereUserId($pid)
->findOrFail($request->input('id'));
$filter->delete();
return redirect(route('settings.privacy.blocked-instances'));
// deprecated
abort(404);
}
public function blockedKeywords()

View file

@ -20,27 +20,43 @@ use App\Jobs\StoryPipeline\StoryViewDeliver;
use App\Services\AccountService;
use App\Services\MediaPathService;
use App\Services\StoryService;
use App\Http\Resources\StoryView as StoryViewResource;
class StoryApiV1Controller extends Controller
{
const RECENT_KEY = 'pf:stories:recent-by-id:';
const RECENT_TTL = 300;
public function carousel(Request $request)
{
abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
$pid = $request->user()->profile_id;
if(config('database.default') == 'pgsql') {
$s = Story::select('stories.*', 'followers.following_id')
$s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
return Story::select('stories.*', 'followers.following_id')
->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
->where('followers.profile_id', $pid)
->where('stories.active', true)
->get();
->map(function($s) {
$r = new \StdClass;
$r->id = $s->id;
$r->profile_id = $s->profile_id;
$r->type = $s->type;
$r->path = $s->path;
return $r;
})
->unique('profile_id');
});
} else {
$s = Story::select('stories.*', 'followers.following_id')
$s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
return Story::select('stories.*', 'followers.following_id')
->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
->where('followers.profile_id', $pid)
->where('stories.active', true)
->orderBy('id')
->get();
});
}
$nodes = $s->map(function($s) use($pid) {
@ -120,6 +136,115 @@ class StoryApiV1Controller extends Controller
return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
}
public function selfCarousel(Request $request)
{
abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
$pid = $request->user()->profile_id;
if(config('database.default') == 'pgsql') {
$s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
return Story::select('stories.*', 'followers.following_id')
->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
->where('followers.profile_id', $pid)
->where('stories.active', true)
->map(function($s) {
$r = new \StdClass;
$r->id = $s->id;
$r->profile_id = $s->profile_id;
$r->type = $s->type;
$r->path = $s->path;
return $r;
})
->unique('profile_id');
});
} else {
$s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
return Story::select('stories.*', 'followers.following_id')
->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
->where('followers.profile_id', $pid)
->where('stories.active', true)
->orderBy('id')
->get();
});
}
$nodes = $s->map(function($s) use($pid) {
$profile = AccountService::get($s->profile_id, true);
if(!$profile || !isset($profile['id'])) {
return false;
}
return [
'id' => (string) $s->id,
'pid' => (string) $s->profile_id,
'type' => $s->type,
'src' => url(Storage::url($s->path)),
'duration' => $s->duration ?? 3,
'seen' => StoryService::hasSeen($pid, $s->id),
'created_at' => $s->created_at->format('c')
];
})
->filter()
->groupBy('pid')
->map(function($item) use($pid) {
$profile = AccountService::get($item[0]['pid'], true);
$url = $profile['local'] ? url("/stories/{$profile['username']}") :
url("/i/rs/{$profile['id']}");
return [
'id' => 'pfs:' . $profile['id'],
'user' => [
'id' => (string) $profile['id'],
'username' => $profile['username'],
'username_acct' => $profile['acct'],
'avatar' => $profile['avatar'],
'local' => $profile['local'],
'is_author' => $profile['id'] == $pid
],
'nodes' => $item,
'url' => $url,
'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
];
})
->sortBy('seen')
->values();
$selfProfile = AccountService::get($pid, true);
$res = [
'self' => [
'user' => [
'id' => (string) $selfProfile['id'],
'username' => $selfProfile['acct'],
'avatar' => $selfProfile['avatar'],
'local' => $selfProfile['local'],
'is_author' => true
],
'nodes' => [],
],
'nodes' => $nodes,
];
if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
$selfStories = Story::whereProfileId($pid)
->whereActive(true)
->get()
->map(function($s) use($pid) {
return [
'id' => (string) $s->id,
'type' => $s->type,
'src' => url(Storage::url($s->path)),
'duration' => $s->duration,
'seen' => true,
'created_at' => $s->created_at->format('c')
];
})
->sortBy('id')
->values();
$res['self']['nodes'] = $selfStories;
}
return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
}
public function add(Request $request)
{
abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
@ -355,4 +480,26 @@ class StoryApiV1Controller extends Controller
$path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
return $path;
}
public function viewers(Request $request)
{
abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
$this->validate($request, [
'sid' => 'required|string|min:1|max:50'
]);
$pid = $request->user()->profile_id;
$sid = $request->input('sid');
$story = Story::whereProfileId($pid)
->whereActive(true)
->findOrFail($sid);
$viewers = StoryView::whereStoryId($story->id)
->orderByDesc('id')
->cursorPaginate(10);
return StoryViewResource::collection($viewers);
}
}

View file

@ -0,0 +1,20 @@
<?php
namespace App\Http\Resources;
use Illuminate\Http\Request;
use Illuminate\Http\Resources\Json\JsonResource;
use App\Services\AccountService;
class StoryView extends JsonResource
{
/**
* Transform the resource into an array.
*
* @return array<string, mixed>
*/
public function toArray(Request $request)
{
return AccountService::get($this->profile_id, true);
}
}

View file

@ -0,0 +1,67 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
AvatarService::cleanup($this->avatar, true);
return;
}
}

View file

@ -0,0 +1,80 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
use Illuminate\Support\Str;
class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
$avatar = $this->avatar;
$disk = AvatarService::disk();
$files = collect(AvatarService::storage($avatar));
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

View file

@ -2,19 +2,25 @@
namespace App\Jobs\AvatarPipeline;
use App\Avatar;
use App\Profile;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Avatar;
use App\Profile;
class CreateAvatar implements ShouldQueue
class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $profile;
public $profile;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* Delete the job if its models no longer exist.
@ -23,6 +29,31 @@ class CreateAvatar implements ShouldQueue
*/
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:create:' . $this->profile->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*
@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
*/
public function __construct(Profile $profile)
{
$this->profile = $profile;
$this->profile = $profile->withoutRelations();
}
/**
@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
public function handle()
{
$profile = $this->profile;
$isRemote = (bool) $profile->private_key == null;
$path = 'public/avatars/default.jpg';
$avatar = new Avatar();
$avatar->profile_id = $profile->id;
$avatar->media_path = $path;
$avatar->change_count = 0;
$avatar->last_processed_at = \Carbon\Carbon::now();
$avatar->save();
Avatar::updateOrCreate(
[
'profile_id' => $profile->id,
],
[
'media_path' => $path,
'change_count' => 0,
'is_remote' => $isRemote,
'last_processed_at' => now()
]
);
}
}

View file

@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
$avatar->remote_url = $icon['url'];
$avatar->save();
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;
}

View file

@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
$avatar->save();
}
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;

View file

@ -57,7 +57,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue
$status = $this->status;
if(AccountService::get($status->profile_id, true)) {
DecrementPostCount::dispatch($status->profile_id)->onQueue('feed');
DecrementPostCount::dispatch($status->profile_id)->onQueue('low');
}
NetworkTimelineService::del($status->id);

View file

@ -0,0 +1,42 @@
<?php
namespace App\Jobs\DirectPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Util\ActivityPub\Helpers;
class DirectDeletePipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
protected $profile;
protected $url;
protected $payload;
/**
* Create a new job instance.
*/
public function __construct($profile, $url, $payload)
{
$this->profile = $profile;
$this->url = $url;
$this->payload = $payload;
}
/**
* Execute the job.
*/
public function handle(): void
{
Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
}
}

View file

@ -0,0 +1,42 @@
<?php
namespace App\Jobs\DirectPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Util\ActivityPub\Helpers;
class DirectDeliverPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
protected $profile;
protected $url;
protected $payload;
/**
* Create a new job instance.
*/
public function __construct($profile, $url, $payload)
{
$this->profile = $profile;
$this->url = $url;
$this->payload = $payload;
}
/**
* Execute the job.
*/
public function handle(): void
{
Helpers::sendSignedObject($this->profile, $this->url, $this->payload);
}
}

View file

@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AccountService;
use App\Services\FollowerService;
use Cache;
use DB;
use Storage;
use App\Follower;
use App\Profile;
class FollowServiceWarmCache implements ShouldQueue
@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue
public $timeout = 5000;
public $failOnTimeout = false;
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping($this->profileId))->dontRelease()];
}
/**
* Create a new job instance.
*
@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue
{
$id = $this->profileId;
if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) {
return;
}
$account = AccountService::get($id, true);
if(!$account) {
@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue
return;
}
DB::table('followers')
->select('id', 'following_id', 'profile_id')
->whereFollowingId($id)
->orderBy('id')
->chunk(200, function($followers) use($id) {
foreach($followers as $follow) {
FollowerService::add($follow->profile_id, $id);
}
});
$hasFollowerPostProcessing = false;
$hasFollowingPostProcessing = false;
DB::table('followers')
->select('id', 'following_id', 'profile_id')
->whereProfileId($id)
->orderBy('id')
->chunk(200, function($followers) use($id) {
foreach($followers as $follow) {
FollowerService::add($id, $follow->following_id);
if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
$following = [];
$followers = [];
foreach(Follower::where('following_id', $id)->orWhere('profile_id', $id)->lazyById(500) as $follow) {
if($follow->following_id != $id && $follow->profile_id != $id) {
continue;
}
if($follow->profile_id == $id) {
$following[] = $follow->following_id;
} else {
$followers[] = $follow->profile_id;
}
}
if(count($followers) > 100) {
// store follower ids and process in another job
Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers));
$hasFollowerPostProcessing = true;
} else {
foreach($followers as $follower) {
FollowerService::add($follower, $id);
}
}
if(count($following) > 100) {
// store following ids and process in another job
Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following));
$hasFollowingPostProcessing = true;
} else {
foreach($following as $following) {
FollowerService::add($id, $following);
}
}
}
});
Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800);
Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800);
@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue
AccountService::del($id);
if($hasFollowingPostProcessing) {
FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow');
}
if($hasFollowerPostProcessing) {
FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow');
}
return;
}
}

View file

@ -0,0 +1,88 @@
<?php
namespace App\Jobs\FollowPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\AccountService;
use App\Services\FollowerService;
use Cache;
use DB;
use Storage;
use App\Follower;
use App\Profile;
class FollowServiceWarmCacheLargeIngestPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $profileId;
public $followType;
public $tries = 5;
public $timeout = 5000;
public $failOnTimeout = false;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($profileId, $followType = 'following')
{
$this->profileId = $profileId;
$this->followType = $followType;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$pid = $this->profileId;
$type = $this->followType;
if($type === 'followers') {
$key = 'follow-warm-cache/' . $pid . '/followers.json';
if(!Storage::exists($key)) {
return;
}
$file = Storage::get($key);
$json = json_decode($file, true);
foreach($json as $id) {
FollowerService::add($id, $pid, false);
usleep(random_int(500, 3000));
}
sleep(5);
Storage::delete($key);
}
if($type === 'following') {
$key = 'follow-warm-cache/' . $pid . '/following.json';
if(!Storage::exists($key)) {
return;
}
$file = Storage::get($key);
$json = json_decode($file, true);
foreach($json as $id) {
FollowerService::add($pid, $id, false);
usleep(random_int(500, 3000));
}
sleep(5);
Storage::delete($key);
}
sleep(random_int(2, 5));
$files = Storage::files('follow-warm-cache/' . $pid);
if(empty($files)) {
Storage::deleteDirectory('follow-warm-cache/' . $pid);
}
}
}

View file

@ -17,6 +17,7 @@ use Illuminate\Support\Facades\Redis;
use App\Services\AccountService;
use App\Services\FollowerService;
use App\Services\NotificationService;
use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
class UnfollowPipeline implements ShouldQueue
{
@ -55,6 +56,8 @@ class UnfollowPipeline implements ShouldQueue
return;
}
FeedUnfollowPipeline::dispatch($actor, $target)->onQueue('follow');
FollowerService::remove($actor, $target);
$actorProfileSync = Cache::get(FollowerService::FOLLOWING_SYNC_KEY . $actor);

View file

@ -0,0 +1,87 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Services\AccountService;
use App\Services\HomeTimelineService;
use App\Services\SnowflakeService;
use App\Status;
class FeedFollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $actorId;
protected $followingId;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:insert:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:insert:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($actorId, $followingId)
{
$this->actorId = $actorId;
$this->followingId = $followingId;
}
/**
* Execute the job.
*/
public function handle(): void
{
$actorId = $this->actorId;
$followingId = $this->followingId;
$minId = SnowflakeService::byDate(now()->subWeeks(6));
$ids = Status::where('id', '>', $minId)
->where('profile_id', $followingId)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereIn('visibility',['public', 'unlisted', 'private'])
->orderByDesc('id')
->limit(HomeTimelineService::FOLLOWER_FEED_POST_LIMIT)
->pluck('id');
foreach($ids as $id) {
HomeTimelineService::add($actorId, $id);
}
}
}

View file

@ -0,0 +1,114 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\UserFilter;
use App\Models\UserDomainBlock;
use App\Services\FollowerService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $sid;
protected $pid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:insert:sid:' . $this->sid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:insert:sid:{$this->sid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($sid, $pid)
{
$this->sid = $sid;
$this->pid = $pid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$sid = $this->sid;
$status = StatusService::get($sid, false);
if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
HomeTimelineService::add($this->pid, $this->sid);
$ids = FollowerService::localFollowerIds($this->pid);
if(!$ids || !count($ids)) {
return;
}
$domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
$skipIds = [];
if(strtolower(config('pixelfed.domain.app')) !== $domain) {
$skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
}
$filters = UserFilter::whereFilterableType('App\Profile')
->whereFilterableId($status['account']['id'])
->whereIn('filter_type', ['mute', 'block'])
->pluck('user_id')
->toArray();
if($filters && count($filters)) {
$skipIds = array_merge($skipIds, $filters);
}
$skipIds = array_unique(array_values($skipIds));
foreach($ids as $id) {
if(!in_array($id, $skipIds)) {
HomeTimelineService::add($id, $this->sid);
}
}
}
}

View file

@ -0,0 +1,112 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\UserFilter;
use App\Models\UserDomainBlock;
use App\Services\FollowerService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $sid;
protected $pid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:insert:remote:sid:' . $this->sid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:insert:remote:sid:{$this->sid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($sid, $pid)
{
$this->sid = $sid;
$this->pid = $pid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$sid = $this->sid;
$status = StatusService::get($sid, false);
if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = FollowerService::localFollowerIds($this->pid);
if(!$ids || !count($ids)) {
return;
}
$domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
$skipIds = [];
if(strtolower(config('pixelfed.domain.app')) !== $domain) {
$skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
}
$filters = UserFilter::whereFilterableType('App\Profile')
->whereFilterableId($status['account']['id'])
->whereIn('filter_type', ['mute', 'block'])
->pluck('user_id')
->toArray();
if($filters && count($filters)) {
$skipIds = array_merge($skipIds, $filters);
}
$skipIds = array_unique(array_values($skipIds));
foreach($ids as $id) {
if(!in_array($id, $skipIds)) {
HomeTimelineService::add($id, $this->sid);
}
}
}
}

View file

@ -0,0 +1,98 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Services\StatusService;
use App\Services\HomeTimelineService;
class FeedRemoveDomainPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $pid;
protected $domain;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:remove:domain:' . $this->pid . ':d-' . $this->domain;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:remove:domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($pid, $domain)
{
$this->pid = $pid;
$this->domain = $domain;
}
/**
* Execute the job.
*/
public function handle(): void
{
if(!config('exp.cached_home_timeline')) {
return;
}
if ($this->batch()->cancelled()) {
return;
}
if(!$this->pid || !$this->domain) {
return;
}
$domain = strtolower($this->domain);
$pid = $this->pid;
$posts = HomeTimelineService::get($pid, '0', '-1');
foreach($posts as $post) {
$status = StatusService::get($post, false);
if(!$status || !isset($status['url'])) {
HomeTimelineService::rem($pid, $post);
continue;
}
$host = strtolower(parse_url($status['url'], PHP_URL_HOST));
if($host === strtolower(config('pixelfed.domain.app')) || !$host) {
continue;
}
if($host === $domain) {
HomeTimelineService::rem($pid, $status['id']);
}
}
}
}

View file

@ -0,0 +1,76 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Services\FollowerService;
use App\Services\StatusService;
use App\Services\HomeTimelineService;
class FeedRemovePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $sid;
protected $pid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:remove:sid:' . $this->sid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:remove:sid:{$this->sid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($sid, $pid)
{
$this->sid = $sid;
$this->pid = $pid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$ids = FollowerService::localFollowerIds($this->pid);
HomeTimelineService::rem($this->pid, $this->sid);
foreach($ids as $id) {
HomeTimelineService::rem($id, $this->sid);
}
}
}

View file

@ -0,0 +1,74 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Services\FollowerService;
use App\Services\StatusService;
use App\Services\HomeTimelineService;
class FeedRemoveRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $sid;
protected $pid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:remove:remote:sid:' . $this->sid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:remove:remote:sid:{$this->sid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($sid, $pid)
{
$this->sid = $sid;
$this->pid = $pid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$ids = FollowerService::localFollowerIds($this->pid);
foreach($ids as $id) {
HomeTimelineService::rem($id, $this->sid);
}
}
}

View file

@ -0,0 +1,81 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Services\AccountService;
use App\Services\StatusService;
use App\Services\HomeTimelineService;
class FeedUnfollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $actorId;
protected $followingId;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hts:feed:remove:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hts:feed:remove:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($actorId, $followingId)
{
$this->actorId = $actorId;
$this->followingId = $followingId;
}
/**
* Execute the job.
*/
public function handle(): void
{
$actorId = $this->actorId;
$followingId = $this->followingId;
$ids = HomeTimelineService::get($actorId, 0, -1);
foreach($ids as $id) {
$status = StatusService::get($id, false);
if($status && isset($status['account'], $status['account']['id'])) {
if($status['account']['id'] == $followingId) {
HomeTimelineService::rem($actorId, $id);
}
}
}
}
}

View file

@ -0,0 +1,67 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\HomeTimelineService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class FeedWarmCachePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $pid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hfp:warm-cache:pid:' . $this->pid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hfp:warm-cache:pid:{$this->pid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($pid)
{
$this->pid = $pid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$pid = $this->pid;
HomeTimelineService::warmCache($pid, true, 400, true);
}
}

View file

@ -0,0 +1,116 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Hashtag;
use App\StatusHashtag;
use App\UserFilter;
use App\Models\UserDomainBlock;
use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $hashtag;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* Delete the job if its models no longer exist.
*
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hfp:hashtag:fanout:insert:' . $this->hashtag->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hfp:hashtag:fanout:insert:{$this->hashtag->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(StatusHashtag $hashtag)
{
$this->hashtag = $hashtag;
}
/**
* Execute the job.
*/
public function handle(): void
{
$hashtag = $this->hashtag;
$sid = $hashtag->status_id;
$status = StatusService::get($sid, false);
if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
$skipIds = [];
if(strtolower(config('pixelfed.domain.app')) !== $domain) {
$skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray();
}
$filters = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
if($filters && count($filters)) {
$skipIds = array_merge($skipIds, $filters);
}
$skipIds = array_unique(array_values($skipIds));
$ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
if(!$ids || !count($ids)) {
return;
}
foreach($ids as $id) {
if(!in_array($id, $skipIds)) {
HomeTimelineService::add($id, $hashtag->status_id);
}
}
}
}

View file

@ -0,0 +1,92 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Hashtag;
use App\StatusHashtag;
use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $sid;
protected $hid;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'hfp:hashtag:fanout:remove:' . $this->hid . ':' . $this->sid;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("hfp:hashtag:fanout:remove:{$this->hid}:{$this->sid}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($sid, $hid)
{
$this->sid = $sid;
$this->hid = $hid;
}
/**
* Execute the job.
*/
public function handle(): void
{
$sid = $this->sid;
$hid = $this->hid;
$status = StatusService::get($sid, false);
if(!$status || !isset($status['account']) || !isset($status['account']['id'])) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = HashtagFollowService::getPidByHid($hid);
if(!$ids || !count($ids)) {
return;
}
foreach($ids as $id) {
HomeTimelineService::rem($id, $sid);
}
}
}

View file

@ -0,0 +1,80 @@
<?php
namespace App\Jobs\HomeFeedPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Support\Facades\Cache;
use App\Follower;
use App\Hashtag;
use App\StatusHashtag;
use App\Services\HashtagFollowService;
use App\Services\StatusService;
use App\Services\HomeTimelineService;
class HashtagUnfollowPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $pid;
protected $hid;
protected $slug;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* Create a new job instance.
*/
public function __construct($hid, $pid, $slug)
{
$this->hid = $hid;
$this->pid = $pid;
$this->slug = $slug;
}
/**
* Execute the job.
*/
public function handle(): void
{
$hid = $this->hid;
$pid = $this->pid;
$slug = strtolower($this->slug);
$statusIds = HomeTimelineService::get($pid, 0, -1);
$followingIds = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
$following = Follower::whereProfileId($pid)->pluck('following_id');
return $following->push($pid)->toArray();
});
foreach($statusIds as $id) {
$status = StatusService::get($id, false);
if(!$status || empty($status['tags'])) {
HomeTimelineService::rem($pid, $id);
continue;
}
$following = in_array((int) $status['account']['id'], $followingIds);
if($following === true) {
continue;
}
$tags = collect($status['tags'])->map(function($tag) {
return strtolower($tag['name']);
})->filter()->values()->toArray();
if(in_array($slug, $tags)) {
HomeTimelineService::rem($pid, $id);
}
}
}
}

View file

@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
class ImageResize implements ShouldQueue
{
@ -46,6 +47,7 @@ class ImageResize implements ShouldQueue
}
$path = storage_path('app/'.$media->media_path);
if (!is_file($path) || $media->skip_optimize) {
Log::info('Tried to optimize media that does not exist or is not readable. ' . $path);
return;
}
@ -57,6 +59,7 @@ class ImageResize implements ShouldQueue
$img = new Image();
$img->resizeImage($media);
} catch (Exception $e) {
Log::error($e);
}
ImageThumbnail::dispatch($media)->onQueue('mmo');

View file

@ -193,7 +193,7 @@ class InboxValidator implements ShouldQueue
}
try {
$res = Http::timeout(20)->withHeaders([
$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
])->get($actor->remote_url);

View file

@ -173,7 +173,7 @@ class InboxWorker implements ShouldQueue
}
try {
$res = Http::timeout(20)->withHeaders([
$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
])->get($actor->remote_url);

View file

@ -0,0 +1,71 @@
<?php
namespace App\Jobs\InternalPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Notification;
use Cache;
use App\Services\NotificationService;
class NotificationEpochUpdatePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 1500;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'ip:notification-epoch-update';
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping('ip:notification-epoch-update'))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct()
{
//
}
/**
* Execute the job.
*/
public function handle(): void
{
$rec = Notification::where('created_at', '>', now()->subMonths(6))->first();
$id = 1;
if($rec) {
$id = $rec->id;
}
Cache::put(NotificationService::EPOCH_CACHE_KEY . '6', $id, 1209600);
}
}

View file

@ -10,8 +10,11 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Storage;
use App\Services\Media\MediaHlsService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class MediaDeletePipeline implements ShouldQueue
class MediaDeletePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
@ -20,8 +23,34 @@ class MediaDeletePipeline implements ShouldQueue
public $timeout = 300;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'media:purge-job:id-' . $this->media->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("media:purge-job:id-{$this->media->id}"))->shared()->dontRelease()];
}
public function __construct(Media $media)
{
$this->media = $media;
@ -63,9 +92,17 @@ class MediaDeletePipeline implements ShouldQueue
$disk->delete($thumb);
}
if($media->hls_path != null) {
$files = MediaHlsService::allFiles($media);
if($files && count($files)) {
foreach($files as $file) {
$disk->delete($file);
}
}
}
$media->delete();
return 1;
}
}

View file

@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue
return 1;
}
if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
$profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
$profile->save();
AccountService::del($id);
} else {
$profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0;
$profile->save();
AccountService::del($id);
}
return 1;
}

View file

@ -8,16 +8,48 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Profile;
use App\Status;
use App\Services\AccountService;
class IncrementPostCount implements ShouldQueue
class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $id;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'propipe:ipc:' . $this->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*
@ -43,17 +75,11 @@ class IncrementPostCount implements ShouldQueue
return 1;
}
if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) {
$profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count();
$profile->last_status_at = now();
$profile->save();
AccountService::del($id);
} else {
$profile->status_count = $profile->status_count + 1;
$profile->last_status_at = now();
$profile->save();
AccountService::del($id);
}
AccountService::get($id);
return 1;
}

View file

@ -0,0 +1,119 @@
<?php
namespace App\Jobs\ProfilePipeline;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Follower;
use App\Profile;
use App\Notification;
use DB;
use App\Services\AccountService;
use App\Services\FollowerService;
use App\Services\NotificationService;
class ProfilePurgeFollowersByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $pid;
protected $domain;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'followers:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("followers:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($pid, $domain)
{
$this->pid = $pid;
$this->domain = $domain;
}
/**
* Execute the job.
*/
public function handle(): void
{
if ($this->batch()->cancelled()) {
return;
}
$pid = $this->pid;
$domain = $this->domain;
$query = 'SELECT f.*
FROM followers f
JOIN profiles p ON p.id = f.profile_id OR p.id = f.following_id
WHERE (f.profile_id = ? OR f.following_id = ?)
AND p.domain = ?;';
$params = [$pid, $pid, $domain];
foreach(DB::cursor($query, $params) as $n) {
if(!$n || !$n->id) {
continue;
}
$follower = Follower::find($n->id);
if($follower->following_id == $pid && $follower->profile_id) {
FollowerService::remove($follower->profile_id, $pid, true);
$follower->delete();
} else if ($follower->profile_id == $pid && $follower->following_id) {
FollowerService::remove($follower->following_id, $pid, true);
$follower->delete();
}
}
$profile = Profile::find($pid);
$followerCount = DB::table('profiles')
->join('followers', 'profiles.id', '=', 'followers.following_id')
->where('followers.following_id', $pid)
->count();
$followingCount = DB::table('profiles')
->join('followers', 'profiles.id', '=', 'followers.following_id')
->where('followers.profile_id', $pid)
->count();
$profile->followers_count = $followerCount;
$profile->following_count = $followingCount;
$profile->save();
AccountService::del($profile->id);
}
}

View file

@ -0,0 +1,91 @@
<?php
namespace App\Jobs\ProfilePipeline;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use App\Notification;
use DB;
use App\Services\NotificationService;
class ProfilePurgeNotificationsByDomain implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $pid;
protected $domain;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'notify:v1:purge-by-domain:' . $this->pid . ':d-' . $this->domain;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("notify:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($pid, $domain)
{
$this->pid = $pid;
$this->domain = $domain;
}
/**
* Execute the job.
*/
public function handle(): void
{
if ($this->batch()->cancelled()) {
return;
}
$pid = $this->pid;
$domain = $this->domain;
$query = 'SELECT notifications.*
FROM profiles
JOIN notifications on profiles.id = notifications.actor_id
WHERE notifications.profile_id = ?
AND profiles.domain = ?';
$params = [$pid, $domain];
foreach(DB::cursor($query, $params) as $n) {
if(!$n || !$n->id) {
continue;
}
Notification::where('id', $n->id)->delete();
NotificationService::del($pid, $n->id);
}
}
}

View file

@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
use App\Util\ActivityPub\HttpSignature;
use App\Services\ReblogService;
use App\Services\StatusService;
use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
class SharePipeline implements ShouldQueue
{
@ -82,6 +83,8 @@ class SharePipeline implements ShouldQueue
]
);
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
return $this->remoteAnnounceDeliver();
}

View file

@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
use App\Util\ActivityPub\HttpSignature;
use App\Services\ReblogService;
use App\Services\StatusService;
use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
class UndoSharePipeline implements ShouldQueue
{
@ -35,6 +36,8 @@ class UndoSharePipeline implements ShouldQueue
$actor = $status->profile;
$parent = Status::find($status->reblog_of_id);
FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
if($parent) {
$target = $parent->profile_id;
ReblogService::removePostReblog($parent->profile_id, $status->id);

View file

@ -21,9 +21,11 @@ use App\{
};
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use League\Fractal;
use Illuminate\Support\Str;
use League\Fractal\Serializer\ArraySerializer;
@ -37,8 +39,10 @@ use App\Services\AccountService;
use App\Services\CollectionService;
use App\Services\StatusService;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
use App\Jobs\ProfilePipeline\DecrementPostCount;
use App\Services\NotificationService;
class RemoteStatusDelete implements ShouldQueue
class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
@ -51,9 +55,35 @@ class RemoteStatusDelete implements ShouldQueue
*/
public $deleteWhenMissingModels = true;
public $timeout = 90;
public $tries = 2;
public $maxExceptions = 1;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 180;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'status:remote:delete:' . $this->status->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
@ -62,7 +92,7 @@ class RemoteStatusDelete implements ShouldQueue
*/
public function __construct(Status $status)
{
$this->status = $status;
$this->status = $status->withoutRelations();
}
/**
@ -77,14 +107,10 @@ class RemoteStatusDelete implements ShouldQueue
if($status->deleted_at) {
return;
}
$profile = $this->status->profile;
StatusService::del($status->id, true);
if($profile->status_count && $profile->status_count > 0) {
$profile->status_count = $profile->status_count - 1;
$profile->save();
}
DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox');
return $this->unlinkRemoveMedia($status);
}
@ -112,14 +138,34 @@ class RemoteStatusDelete implements ShouldQueue
CollectionService::removeItem($col->collection_id, $col->object_id);
$col->delete();
});
DirectMessage::whereStatusId($status->id)->delete();
$dms = DirectMessage::whereStatusId($status->id)->get();
foreach($dms as $dm) {
$not = Notification::whereItemType('App\DirectMessage')
->whereItemId($dm->id)
->first();
if($not) {
NotificationService::del($not->profile_id, $not->id);
$not->forceDeleteQuietly();
}
$dm->delete();
}
Like::whereStatusId($status->id)->forceDelete();
Media::whereStatusId($status->id)
->get()
->each(function($media) {
MediaDeletePipeline::dispatch($media)->onQueue('mmo');
});
MediaTag::where('status_id', $status->id)->delete();
$mediaTags = MediaTag::where('status_id', $status->id)->get();
foreach($mediaTags as $mtag) {
$not = Notification::whereItemType('App\MediaTag')
->whereItemId($mtag->id)
->first();
if($not) {
NotificationService::del($not->profile_id, $not->id);
$not->forceDeleteQuietly();
}
$mtag->delete();
}
Mention::whereStatusId($status->id)->forceDelete();
Notification::whereItemType('App\Status')
->whereItemId($status->id)

View file

@ -35,6 +35,7 @@ use GuzzleHttp\Promise;
use App\Util\ActivityPub\HttpSignature;
use App\Services\CollectionService;
use App\Services\StatusService;
use App\Services\NotificationService;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
class StatusDelete implements ShouldQueue
@ -115,10 +116,30 @@ class StatusDelete implements ShouldQueue
$col->delete();
});
DirectMessage::whereStatusId($status->id)->delete();
$dms = DirectMessage::whereStatusId($status->id)->get();
foreach($dms as $dm) {
$not = Notification::whereItemType('App\DirectMessage')
->whereItemId($dm->id)
->first();
if($not) {
NotificationService::del($not->profile_id, $not->id);
$not->forceDeleteQuietly();
}
$dm->delete();
}
Like::whereStatusId($status->id)->delete();
MediaTag::where('status_id', $status->id)->delete();
$mediaTags = MediaTag::where('status_id', $status->id)->get();
foreach($mediaTags as $mtag) {
$not = Notification::whereItemType('App\MediaTag')
->whereItemId($mtag->id)
->first();
if($not) {
NotificationService::del($not->profile_id, $not->id);
$not->forceDeleteQuietly();
}
$mtag->delete();
}
Mention::whereStatusId($status->id)->forceDelete();
Notification::whereItemType('App\Status')

View file

@ -19,7 +19,11 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\StatusService;
use App\Services\UserFilterService;
use App\Services\AdminShadowFilterService;
use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
class StatusEntityLexer implements ShouldQueue
{
@ -104,12 +108,12 @@ class StatusEntityLexer implements ShouldQueue
}
DB::transaction(function () use ($status, $tag) {
$slug = str_slug($tag, '-', false);
$hashtag = Hashtag::where('slug', $slug)->first();
if (!$hashtag) {
$hashtag = Hashtag::create(
['name' => $tag, 'slug' => $slug]
);
}
$hashtag = Hashtag::firstOrCreate([
'slug' => $slug
], [
'name' => $tag
]);
StatusHashtag::firstOrCreate(
[
@ -149,6 +153,21 @@ class StatusEntityLexer implements ShouldQueue
MentionPipeline::dispatch($status, $m);
});
}
$this->fanout();
}
public function fanout()
{
$status = $this->status;
StatusService::refresh($status->id);
if(config('exp.cached_home_timeline')) {
if( $status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private'])
) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}
$this->deliver();
}
@ -176,8 +195,10 @@ class StatusEntityLexer implements ShouldQueue
$status->reblog_of_id === null &&
($hideNsfw ? $status->is_nsfw == false : true)
) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
PublicTimelineService::add($status->id);
}
}
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
StatusActivityPubDeliver::dispatch($status);

View file

@ -90,7 +90,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
]);
$nm->each(function($n, $key) use($status) {
$res = Http::retry(3, 100, throw: false)->head($n['url']);
$res = Http::withOptions(['allow_redirects' => false])->retry(3, 100, throw: false)->head($n['url']);
if(!$res->successful()) {
return;

View file

@ -45,6 +45,11 @@ class StatusTagsPipeline implements ShouldQueue
{
$res = $this->activity;
$status = $this->status;
if(isset($res['tag']['type'], $res['tag']['name'])) {
$res['tag'] = [$res['tag']];
}
$tags = collect($res['tag']);
// Emoji
@ -73,19 +78,18 @@ class StatusTagsPipeline implements ShouldQueue
if(config('database.default') === 'pgsql') {
$hashtag = Hashtag::where('name', 'ilike', $name)
->orWhere('slug', 'ilike', str_slug($name))
->orWhere('slug', 'ilike', str_slug($name, '-', false))
->first();
if(!$hashtag) {
$hashtag = new Hashtag;
$hashtag->name = $name;
$hashtag->slug = str_slug($name);
$hashtag->save();
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
}
} else {
$hashtag = Hashtag::firstOrCreate([
'slug' => str_slug($name)
], [
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
}
@ -128,5 +132,7 @@ class StatusTagsPipeline implements ShouldQueue
$mention->save();
MentionPipeline::dispatch($status, $mention);
});
StatusService::refresh($status->id);
}
}

View file

@ -0,0 +1,109 @@
<?php
namespace App\Jobs\VideoPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use FFMpeg\Format\Video\X264;
use FFMpeg;
use Cache;
use App\Services\MediaService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class VideoHlsPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $media;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'media:video-hls:id-' . $this->media->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("media:video-hls:id-{$this->media->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct($media)
{
$this->media = $media;
}
/**
* Execute the job.
*/
public function handle(): void
{
$depCheck = Cache::rememberForever('video-pipeline:hls:depcheck', function() {
$bin = config('laravel-ffmpeg.ffmpeg.binaries');
$output = shell_exec($bin . ' -version');
if($output && preg_match('/ffmpeg version ([^\s]+)/', $output, $matches)) {
$version = $matches[1];
return (version_compare($version, config('laravel-ffmpeg.min_hls_version')) >= 0) ? 'ok' : false;
} else {
return false;
}
});
if(!$depCheck || $depCheck !== 'ok') {
return;
}
$media = $this->media;
$bitrate = (new X264)->setKiloBitrate(config('media.hls.bitrate') ?? 1000);
$mp4 = $media->media_path;
$man = str_replace('.mp4', '.m3u8', $mp4);
FFMpeg::fromDisk('local')
->open($mp4)
->exportForHLS()
->setSegmentLength(16)
->setKeyFrameInterval(48)
->addFormat($bitrate)
->save($man);
$media->hls_path = $man;
$media->hls_transcoded_at = now();
$media->save();
MediaService::del($media->status_id);
usleep(50000);
StatusService::del($media->status_id);
return;
}
}

View file

@ -16,13 +16,46 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline;
use App\Util\Media\Blurhash;
use App\Services\MediaService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
class VideoThumbnail implements ShouldQueue
class VideoThumbnail implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $media;
public $timeout = 900;
public $tries = 3;
public $maxExceptions = 1;
public $failOnTimeout = true;
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'media:video-thumb:id-' . $this->media->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("media:video-thumb:id-{$this->media->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*
@ -54,7 +87,7 @@ class VideoThumbnail implements ShouldQueue
$path[$i] = $t;
$save = implode('/', $path);
$video = FFMpeg::open($base)
->getFrameFromSeconds(0)
->getFrameFromSeconds(1)
->export()
->toDisk('local')
->save($save);
@ -68,6 +101,9 @@ class VideoThumbnail implements ShouldQueue
$media->save();
}
if(config('media.hls.enabled')) {
VideoHlsPipeline::dispatch($media)->onQueue('mmo');
}
} catch (Exception $e) {
}

View file

@ -0,0 +1,33 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use App\Services\AccountService;
use App\Profile;
class AdminShadowFilter extends Model
{
use HasFactory;
protected $guarded = [];
protected $casts = [
'created_at' => 'datetime'
];
public function account()
{
if($this->item_type === 'App\Profile') {
return AccountService::get($this->item_id, true);
}
return;
}
public function profile()
{
return $this->belongsTo(Profile::class, 'item_id');
}
}

View file

@ -0,0 +1,13 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class DefaultDomainBlock extends Model
{
use HasFactory;
protected $guarded = [];
}

View file

@ -0,0 +1,24 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class HashtagRelated extends Model
{
use HasFactory;
protected $guarded = [];
/**
* The attributes that should be mutated to dates and other custom formats.
*
* @var array
*/
protected $casts = [
'related_tags' => 'array',
'last_calculated_at' => 'datetime',
'last_moderated_at' => 'datetime',
];
}

View file

@ -0,0 +1,21 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use App\Profile;
class UserDomainBlock extends Model
{
use HasFactory;
protected $guarded = [];
public $timestamps = false;
public function profile()
{
return $this->belongsTo(Profile::class, 'profile_id');
}
}

View file

@ -5,6 +5,8 @@ namespace App\Observers;
use App\Follower;
use App\Services\FollowerService;
use Cache;
use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
class FollowerObserver
{
@ -21,6 +23,7 @@ class FollowerObserver
}
FollowerService::add($follower->profile_id, $follower->following_id);
FeedFollowPipeline::dispatch($follower->profile_id, $follower->following_id)->onQueue('follow');
}
/**

View file

@ -0,0 +1,51 @@
<?php
namespace App\Observers;
use App\HashtagFollow;
use App\Services\HashtagFollowService;
use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
{
/**
* Handle the HashtagFollow "created" event.
*/
public function created(HashtagFollow $hashtagFollow): void
{
HashtagFollowService::add($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
}
/**
* Handle the HashtagFollow "updated" event.
*/
public function updated(HashtagFollow $hashtagFollow): void
{
//
}
/**
* Handle the HashtagFollow "deleting" event.
*/
public function deleting(HashtagFollow $hashtagFollow): void
{
HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
}
/**
* Handle the HashtagFollow "restored" event.
*/
public function restored(HashtagFollow $hashtagFollow): void
{
//
}
/**
* Handle the HashtagFollow "force deleted" event.
*/
public function forceDeleted(HashtagFollow $hashtagFollow): void
{
HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
}
}

View file

@ -5,32 +5,31 @@ namespace App\Observers;
use DB;
use App\StatusHashtag;
use App\Services\StatusHashtagService;
use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
use App\Jobs\HomeFeedPipeline\HashtagRemoveFanoutPipeline;
use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
class StatusHashtagObserver
class StatusHashtagObserver implements ShouldHandleEventsAfterCommit
{
/**
* Handle events after all transactions are committed.
*
* @var bool
*/
public $afterCommit = true;
/**
* Handle the notification "created" event.
*
* @param \App\Notification $notification
* @param \App\StatusHashtag $hashtag
* @return void
*/
public function created(StatusHashtag $hashtag)
{
StatusHashtagService::set($hashtag->hashtag_id, $hashtag->status_id);
DB::table('hashtags')->where('id', $hashtag->hashtag_id)->increment('cached_count');
if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
HashtagInsertFanoutPipeline::dispatch($hashtag)->onQueue('feed');
}
}
/**
* Handle the notification "updated" event.
*
* @param \App\Notification $notification
* @param \App\StatusHashtag $hashtag
* @return void
*/
public function updated(StatusHashtag $hashtag)
@ -41,19 +40,22 @@ class StatusHashtagObserver
/**
* Handle the notification "deleted" event.
*
* @param \App\Notification $notification
* @param \App\StatusHashtag $hashtag
* @return void
*/
public function deleted(StatusHashtag $hashtag)
{
StatusHashtagService::del($hashtag->hashtag_id, $hashtag->status_id);
DB::table('hashtags')->where('id', $hashtag->hashtag_id)->decrement('cached_count');
if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
HashtagRemoveFanoutPipeline::dispatch($hashtag->status_id, $hashtag->hashtag_id)->onQueue('feed');
}
}
/**
* Handle the notification "restored" event.
*
* @param \App\Notification $notification
* @param \App\StatusHashtag $hashtag
* @return void
*/
public function restored(StatusHashtag $hashtag)
@ -64,7 +66,7 @@ class StatusHashtagObserver
/**
* Handle the notification "force deleted" event.
*
* @param \App\Notification $notification
* @param \App\StatusHashtag $hashtag
* @return void
*/
public function forceDeleted(StatusHashtag $hashtag)

View file

@ -7,6 +7,8 @@ use App\Services\ProfileStatusService;
use Cache;
use App\Models\ImportPost;
use App\Services\ImportService;
use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
class StatusObserver
{
@ -63,6 +65,14 @@ class StatusObserver
ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete();
ImportService::clearImportedFiles($status->profile_id);
}
if(config('exp.cached_home_timeline')) {
if($status->uri) {
FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
} else {
FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}
}
/**

View file

@ -4,6 +4,8 @@ namespace App\Observers;
use App\UserFilter;
use App\Services\UserFilterService;
use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
class UserFilterObserver
{
@ -78,10 +80,12 @@ class UserFilterObserver
switch ($userFilter->filter_type) {
case 'mute':
UserFilterService::mute($userFilter->user_id, $userFilter->filterable_id);
FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
break;
case 'block':
UserFilterService::block($userFilter->user_id, $userFilter->filterable_id);
FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
break;
}
}
@ -96,10 +100,12 @@ class UserFilterObserver
switch ($userFilter->filter_type) {
case 'mute':
UserFilterService::unmute($userFilter->user_id, $userFilter->filterable_id);
FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
break;
case 'block':
UserFilterService::unblock($userFilter->user_id, $userFilter->filterable_id);
FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
break;
}
}

View file

@ -7,6 +7,9 @@ use App\Follower;
use App\Profile;
use App\User;
use App\UserSetting;
use App\Services\UserFilterService;
use App\Models\DefaultDomainBlock;
use App\Models\UserDomainBlock;
use App\Jobs\FollowPipeline\FollowPipeline;
use DB;
use App\Services\FollowerService;
@ -14,7 +17,18 @@ use App\Services\FollowerService;
class UserObserver
{
/**
* Listen to the User created event.
* Handle the notification "created" event.
*
* @param \App\User $user
* @return void
*/
public function created(User $user): void
{
$this->handleUser($user);
}
/**
* Listen to the User saved event.
*
* @param \App\User $user
*
@ -22,7 +36,38 @@ class UserObserver
*/
public function saved(User $user)
{
if($user->status == 'deleted') {
$this->handleUser($user);
}
/**
* Listen to the User updated event.
*
* @param \App\User $user
*
* @return void
*/
public function updated(User $user): void
{
$this->handleUser($user);
if($user->profile) {
$this->applyDefaultDomainBlocks($user);
}
}
/**
* Handle the user "deleted" event.
*
* @param \App\User $user
* @return void
*/
public function deleted(User $user)
{
FollowerService::delCache($user->profile_id);
}
protected function handleUser($user)
{
if(in_array($user->status, ['deleted', 'delete'])) {
return;
}
@ -49,9 +94,11 @@ class UserObserver
$profile->private_key = $pki_private;
$profile->public_key = $pki_public;
$profile->save();
$this->applyDefaultDomainBlocks($user);
return $profile;
});
DB::transaction(function() use($user, $profile) {
$user = User::findOrFail($user->id);
$user->profile_id = $profile->id;
@ -92,14 +139,22 @@ class UserObserver
}
}
/**
* Handle the user "deleted" event.
*
* @param \App\User $user
* @return void
*/
public function deleted(User $user)
protected function applyDefaultDomainBlocks($user)
{
FollowerService::delCache($user->profile_id);
if($user->profile_id == null) {
return;
}
$defaultDomainBlocks = DefaultDomainBlock::pluck('domain')->toArray();
if(!$defaultDomainBlocks || !count($defaultDomainBlocks)) {
return;
}
foreach($defaultDomainBlocks as $domain) {
UserDomainBlock::updateOrCreate([
'profile_id' => $user->profile_id,
'domain' => strtolower(trim($domain))
]);
}
}
}

View file

@ -7,6 +7,7 @@ use App\Profile;
use App\Status;
use App\User;
use App\UserSetting;
use App\Models\UserDomainBlock;
use App\Transformer\Api\AccountTransformer;
use League\Fractal;
use League\Fractal\Serializer\ArraySerializer;
@ -166,7 +167,7 @@ class AccountService
public static function usernameToId($username)
{
$key = self::CACHE_KEY . 'u2id:' . hash('sha256', $username);
return Cache::remember($key, 900, function() use($username) {
return Cache::remember($key, 14400, function() use($username) {
$s = Str::of($username);
if($s->contains('@') && !$s->startsWith('@')) {
$username = "@{$username}";
@ -220,4 +221,27 @@ class AccountService
return false;
});
}
public static function setLastActive($id = false)
{
if(!$id) { return; }
$key = 'user:last_active_at:id:' . $id;
if(!Cache::has($key)) {
$user = User::find($id);
if(!$user) { return; }
$user->last_active_at = now();
$user->save();
Cache::put($key, 1, 14400);
}
return;
}
public static function blocksDomain($pid, $domain = false)
{
if(!$domain) {
return;
}
return UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->exists();
}
}

View file

@ -28,7 +28,7 @@ class ActivityPubFetchService
$headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
try {
$res = Http::withHeaders($headers)
$res = Http::withOptions(['allow_redirects' => false])->withHeaders($headers)
->timeout(30)
->connectTimeout(5)
->retry(3, 500)

View file

@ -0,0 +1,51 @@
<?php
namespace App\Services;
use App\Models\AdminShadowFilter;
use Cache;
class AdminShadowFilterService
{
const CACHE_KEY = 'pf:services:asfs:';
public static function queryFilter($name = 'hide_from_public_feeds')
{
return AdminShadowFilter::whereItemType('App\Profile')
->whereActive(1)
->where('hide_from_public_feeds', true)
->pluck('item_id')
->toArray();
}
public static function getHideFromPublicFeedsList($refresh = false)
{
$key = self::CACHE_KEY . 'list:hide_from_public_feeds';
if($refresh) {
Cache::forget($key);
}
return Cache::remember($key, 86400, function() {
return AdminShadowFilter::whereItemType('App\Profile')
->whereActive(1)
->where('hide_from_public_feeds', true)
->pluck('item_id')
->toArray();
});
}
public static function canAddToPublicFeedByProfileId($profileId)
{
return !in_array($profileId, self::getHideFromPublicFeedsList());
}
public static function refresh()
{
$keys = [
self::CACHE_KEY . 'list:hide_from_public_feeds'
];
foreach($keys as $key) {
Cache::forget($key);
}
}
}

View file

@ -3,7 +3,13 @@
namespace App\Services;
use Cache;
use Storage;
use Illuminate\Support\Str;
use App\Avatar;
use App\Profile;
use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
use League\Flysystem\UnableToCheckDirectoryExistence;
use League\Flysystem\UnableToRetrieveMetadata;
class AvatarService
{
@ -20,4 +26,102 @@ class AvatarService
}
return $profile->avatarUrl();
}
public static function disk()
{
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return false;
}
$driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
$disk = Storage::disk($driver);
return $disk;
}
public static function storage(Avatar $avatar)
{
$disk = self::disk();
if(!$disk) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
return $disk->allFiles($base . $avatar->profile_id);
}
public static function cleanup($avatar, $confirm = false)
{
if(!$avatar || !$confirm) {
return;
}
if($avatar->cdn_url == null) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return;
}
$disk = self::disk();
if(!$disk) {
return;
}
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
try {
$exists = $disk->directoryExists($base . $avatar->profile_id);
} catch (
UnableToRetrieveMetadata |
UnableToCheckDirectoryExistence |
Exception $e
) {
return;
}
if(!$exists) {
return;
}
$files = collect($disk->allFiles($base . $avatar->profile_id));
if(!$files || !$files->count() || $files->count() === 1) {
return;
}
if($files->count() > 5) {
AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
return;
}
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

View file

@ -19,26 +19,34 @@ class FollowerService
const FOLLOWING_SYNC_KEY = 'pf:services:followers:sync-following:';
const FOLLOWING_KEY = 'pf:services:follow:following:id:';
const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
const FOLLOWERS_LOCAL_KEY = 'pf:services:follow:local-follower-ids:';
const FOLLOWERS_INTER_KEY = 'pf:services:follow:followers:inter:id:';
public static function add($actor, $target)
public static function add($actor, $target, $refresh = true)
{
$ts = (int) microtime(true);
if($refresh) {
RelationshipService::refresh($actor, $target);
} else {
RelationshipService::forget($actor, $target);
}
Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target);
Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor);
Cache::forget('profile:following:' . $actor);
}
public static function remove($actor, $target)
public static function remove($actor, $target, $silent = false)
{
Redis::zrem(self::FOLLOWING_KEY . $actor, $target);
Redis::zrem(self::FOLLOWERS_KEY . $target, $actor);
Cache::forget('pf:services:follower:audience:' . $actor);
Cache::forget('pf:services:follower:audience:' . $target);
if($silent !== true) {
AccountService::del($actor);
AccountService::del($target);
RelationshipService::refresh($actor, $target);
Cache::forget('profile:following:' . $actor);
} else {
RelationshipService::forget($actor, $target);
}
}
public static function followers($id, $start = 0, $stop = 10)
@ -83,12 +91,16 @@ class FollowerService
return Redis::zCard(self::FOLLOWING_KEY . $id);
}
public static function follows(string $actor, string $target)
public static function follows(string $actor, string $target, $quickCheck = false)
{
if($actor == $target) {
return false;
}
if($quickCheck) {
return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
}
if(self::followerCount($target, false) && self::followingCount($actor, false)) {
self::cacheSyncCheck($target, 'followers');
return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor);
@ -200,6 +212,22 @@ class FollowerService
});
}
public static function mutualAccounts($actorId, $profileId)
{
if($actorId == $profileId) {
return [];
}
$actorKey = self::FOLLOWING_KEY . $actorId;
$profileKey = self::FOLLOWERS_KEY . $profileId;
$key = self::FOLLOWERS_INTER_KEY . $actorId . ':' . $profileId;
$res = Redis::zinterstore($key, [$actorKey, $profileKey]);
if($res) {
return Redis::zrange($key, 0, -1);
} else {
return [];
}
}
public static function delCache($id)
{
Redis::del(self::CACHE_KEY . $id);
@ -208,4 +236,15 @@ class FollowerService
Cache::forget(self::FOLLOWERS_SYNC_KEY . $id);
Cache::forget(self::FOLLOWING_SYNC_KEY . $id);
}
public static function localFollowerIds($pid, $limit = 0)
{
$key = self::FOLLOWERS_LOCAL_KEY . $pid;
$res = Cache::remember($key, 7200, function() use($pid) {
return DB::table('followers')->whereFollowingId($pid)->whereLocalProfile(true)->pluck('profile_id')->sort();
});
return $limit ?
$res->take($limit)->values()->toArray() :
$res->values()->toArray();
}
}

View file

@ -0,0 +1,72 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use App\Hashtag;
use App\StatusHashtag;
use App\HashtagFollow;
class HashtagFollowService
{
const FOLLOW_KEY = 'pf:services:hashtag-follows:v1:';
const CACHE_KEY = 'pf:services:hfs:byHid:';
const CACHE_WARMED = 'pf:services:hfs:wc:byHid';
public static function getPidByHid($hid)
{
if(!self::isWarm($hid)) {
return self::warmCache($hid);
}
return self::get($hid);
}
public static function unfollow($hid, $pid)
{
return Redis::zrem(self::CACHE_KEY . $hid, $pid);
}
public static function add($hid, $pid)
{
return Redis::zadd(self::CACHE_KEY . $hid, $pid, $pid);
}
public static function rem($hid, $pid)
{
return Redis::zrem(self::CACHE_KEY . $hid, $pid);
}
public static function get($hid)
{
return Redis::zrange(self::CACHE_KEY . $hid, 0, -1);
}
public static function count($hid)
{
return Redis::zcard(self::CACHE_KEY . $hid);
}
public static function warmCache($hid)
{
foreach(HashtagFollow::whereHashtagId($hid)->lazyById(20, 'id') as $h) {
if($h) {
self::add($h->hashtag_id, $h->profile_id);
}
}
self::setWarm($hid);
return self::get($hid);
}
public static function isWarm($hid)
{
return Redis::zcount(self::CACHE_KEY . $hid, 0, -1) ?? Redis::zscore(self::CACHE_WARMED, $hid) != null;
}
public static function setWarm($hid)
{
return Redis::zadd(self::CACHE_WARMED, $hid, $hid);
}
}

View file

@ -0,0 +1,38 @@
<?php
namespace App\Services;
use DB;
use App\StatusHashtag;
use App\Models\HashtagRelated;
class HashtagRelatedService
{
public static function get($id)
{
$tag = HashtagRelated::whereHashtagId($id)->first();
if(!$tag) {
return [];
}
return $tag->related_tags;
}
public static function fetchRelatedTags($tag)
{
$res = StatusHashtag::query()
->select('h2.name', DB::raw('COUNT(*) as related_count'))
->join('status_hashtags as hs2', function ($join) {
$join->on('status_hashtags.status_id', '=', 'hs2.status_id')
->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
})
->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
->where('h1.name', '=', $tag)
->groupBy('h2.name')
->orderBy('related_count', 'desc')
->limit(30)
->get();
return $res;
}
}

View file

@ -8,9 +8,10 @@ use App\Hashtag;
use App\StatusHashtag;
use App\HashtagFollow;
class HashtagService {
const FOLLOW_KEY = 'pf:services:hashtag:following:';
class HashtagService
{
const FOLLOW_KEY = 'pf:services:hashtag:following:v1:';
const FOLLOW_PIDS_KEY = 'pf:services:hashtag-follows:v1:';
public static function get($id)
{
@ -28,28 +29,29 @@ class HashtagService {
public static function count($id)
{
return Cache::remember('services:hashtag:public-count:by_id:' . $id, 86400, function() use($id) {
return StatusHashtag::whereHashtagId($id)->whereStatusVisibility('public')->count();
return Cache::remember('services:hashtag:total-count:by_id:' . $id, 300, function() use($id) {
$tag = Hashtag::find($id);
return $tag ? $tag->cached_count ?? 0 : 0;
});
}
public static function isFollowing($pid, $hid)
{
$res = Redis::zscore(self::FOLLOW_KEY . $pid, $hid);
$res = Redis::zscore(self::FOLLOW_KEY . $hid, $pid);
if($res) {
return true;
}
$synced = Cache::get(self::FOLLOW_KEY . $pid . ':synced');
$synced = Cache::get(self::FOLLOW_KEY . 'acct:' . $pid . ':synced');
if(!$synced) {
$tags = HashtagFollow::whereProfileId($pid)
->get()
->each(function($tag) use($pid) {
self::follow($pid, $tag->hashtag_id);
});
Cache::set(self::FOLLOW_KEY . $pid . ':synced', true, 1209600);
Cache::set(self::FOLLOW_KEY . 'acct:' . $pid . ':synced', true, 1209600);
return (bool) Redis::zscore(self::FOLLOW_KEY . $pid, $hid) > 1;
return (bool) Redis::zscore(self::FOLLOW_KEY . $hid, $pid) >= 1;
}
return false;
@ -57,16 +59,29 @@ class HashtagService {
public static function follow($pid, $hid)
{
return Redis::zadd(self::FOLLOW_KEY . $pid, $hid, $hid);
Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
return Redis::zadd(self::FOLLOW_KEY . $hid, $pid, $pid);
}
public static function unfollow($pid, $hid)
{
return Redis::zrem(self::FOLLOW_KEY . $pid, $hid);
Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
return Redis::zrem(self::FOLLOW_KEY . $hid, $pid);
}
public static function following($pid, $start = 0, $limit = 10)
public static function following($hid, $start = 0, $limit = 10)
{
return Redis::zrevrange(self::FOLLOW_KEY . $pid, $start, $limit);
$synced = Cache::get(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced');
if(!$synced) {
$tags = HashtagFollow::whereHashtagId($hid)
->get()
->each(function($tag) use($hid) {
self::follow($tag->profile_id, $hid);
});
Cache::set(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced', true, 1209600);
return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
}
return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
}
}

View file

@ -0,0 +1,114 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use App\Follower;
use App\Status;
use App\Models\UserDomainBlock;
class HomeTimelineService
{
const CACHE_KEY = 'pf:services:timeline:home:';
const FOLLOWER_FEED_POST_LIMIT = 10;
public static function get($id, $start = 0, $stop = 10)
{
if($stop > 100) {
$stop = 100;
}
return Redis::zrevrange(self::CACHE_KEY . $id, $start, $stop);
}
public static function getRankedMaxId($id, $start = null, $limit = 10)
{
if(!$start) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit - 1]
]));
}
public static function getRankedMinId($id, $end = null, $limit = 10)
{
if(!$end) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}
public static function add($id, $val)
{
if(self::count($id) >= 400) {
Redis::zpopmin(self::CACHE_KEY . $id);
}
return Redis::zadd(self::CACHE_KEY .$id, $val, $val);
}
public static function rem($id, $val)
{
return Redis::zrem(self::CACHE_KEY . $id, $val);
}
public static function count($id)
{
return Redis::zcard(self::CACHE_KEY . $id);
}
public static function warmCache($id, $force = false, $limit = 100, $returnIds = false)
{
if(self::count($id) == 0 || $force == true) {
Redis::del(self::CACHE_KEY . $id);
$following = Cache::remember('profile:following:'.$id, 1209600, function() use($id) {
$following = Follower::whereProfileId($id)->pluck('following_id');
return $following->push($id)->toArray();
});
$minId = SnowflakeService::byDate(now()->subMonths(6));
$filters = UserFilterService::filters($id);
if($filters && count($filters)) {
$following = array_diff($following, $filters);
}
$domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray();
$ids = Status::where('id', '>', $minId)
->whereIn('profile_id', $following)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereIn('visibility',['public', 'unlisted', 'private'])
->orderByDesc('id')
->limit($limit)
->pluck('id');
foreach($ids as $pid) {
$status = StatusService::get($pid, false);
if(!$status || !isset($status['account'], $status['url'])) {
continue;
}
if($domainBlocks && count($domainBlocks)) {
$domain = strtolower(parse_url($status['url'], PHP_URL_HOST));
if(in_array($domain, $domainBlocks)) {
continue;
}
}
self::add($id, $pid);
}
return $returnIds ? $ids : 1;
}
return 0;
}
}

View file

@ -120,6 +120,9 @@ class InstanceService
$pixels[] = $row;
}
// Free the allocated GdImage object from memory:
imagedestroy($image);
$components_x = 4;
$components_y = 4;
$blurhash = Blurhash::encode($pixels, $components_x, $components_y);

View file

@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis;
use App\Status;
use App\User;
use App\Services\AccountService;
use App\Util\Site\Nodeinfo;
class LandingService
{
public static function get($json = true)
{
$activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() {
return User::select('last_active_at')
->where('last_active_at', '>', now()->subMonths(1))
->orWhere('created_at', '>', now()->subMonths(1))
->count();
});
$activeMonth = Nodeinfo::activeUsersMonthly();
$totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() {
return User::count();

View file

@ -13,7 +13,7 @@ class MarkerService
return Cache::get(self::CACHE_KEY . $timeline . ':' . $profileId);
}
public static function set($profileId, $timeline = 'home', $entityId)
public static function set($profileId, $timeline = 'home', $entityId = false)
{
$existing = self::get($profileId, $timeline);
$key = self::CACHE_KEY . $timeline . ':' . $profileId;

View file

@ -0,0 +1,27 @@
<?php
namespace App\Services\Media;
use Storage;
class MediaHlsService
{
public static function allFiles($media)
{
$path = $media->media_path;
if(!$path) { return; }
$parts = explode('/', $path);
$filename = array_pop($parts);
$dir = implode('/', $parts);
[$name, $ext] = explode('.', $filename);
$files = Storage::files($dir);
return collect($files)
->filter(function($p) use($dir, $name) {
return str_starts_with($p, $dir . '/' . $name);
})
->values()
->toArray();
}
}

View file

@ -18,7 +18,7 @@ class MediaService
public static function get($statusId)
{
return Cache::remember(self::CACHE_KEY.$statusId, 86400, function() use($statusId) {
return Cache::remember(self::CACHE_KEY.$statusId, 21600, function() use($statusId) {
$media = Media::whereStatusId($statusId)->orderBy('order')->get();
if(!$media) {
return [];
@ -46,7 +46,8 @@ class MediaService
$media['orientation'],
$media['filter_name'],
$media['filter_class'],
$media['mime']
$media['mime'],
$media['hls_manifest']
);
$media['type'] = $mime ? strtolower($mime[0]) : 'unknown';

View file

@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
use GuzzleHttp\Exception\RequestException;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
use Illuminate\Support\Arr;
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
class MediaStorageService {
@ -29,9 +30,9 @@ class MediaStorageService {
return;
}
public static function avatar($avatar, $local = false)
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
{
return (new self())->fetchAvatar($avatar, $local);
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
}
public static function head($url)
@ -86,12 +87,11 @@ class MediaStorageService {
$thumbname = array_pop($pt);
$storagePath = implode('/', $p);
$disk = Storage::disk(config('filesystems.cloud'));
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
$url = $disk->url($file);
$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
$thumbUrl = $disk->url($thumbFile);
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
if($thumb) {
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
$media->thumbnail_url = $thumbUrl;
}
$media->cdn_url = $url;
$media->optimized_url = $url;
$media->replicated_at = now();
@ -183,6 +183,7 @@ class MediaStorageService {
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
{
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
$url = $avatar->remote_url;
$driver = $local ? 'local' : config('filesystems.cloud');
@ -206,7 +207,7 @@ class MediaStorageService {
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
if(!$skipRecentCheck) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
return;
}
}
@ -262,6 +263,7 @@ class MediaStorageService {
Cache::forget('avatar:' . $avatar->profile_id);
AccountService::del($avatar->profile_id);
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
unlink($tmpName);
}

View file

@ -12,10 +12,13 @@ use App\Transformer\Api\NotificationTransformer;
use League\Fractal;
use League\Fractal\Serializer\ArraySerializer;
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
class NotificationService {
const CACHE_KEY = 'pf:services:notifications:ids:';
const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:';
const ITEM_CACHE_TTL = 86400;
const MASTODON_TYPES = [
'follow',
'follow_request',
@ -44,11 +47,22 @@ class NotificationService {
return $res;
}
public static function getEpochId($months = 6)
{
$epoch = Cache::get(self::EPOCH_CACHE_KEY . $months);
if(!$epoch) {
NotificationEpochUpdatePipeline::dispatch();
return 1;
}
return $epoch;
}
public static function coldGet($id, $start = 0, $stop = 400)
{
$stop = $stop > 400 ? 400 : $stop;
$ids = Notification::whereProfileId($id)
->latest()
$ids = Notification::where('id', '>', self::getEpochId())
->where('profile_id', $id)
->orderByDesc('id')
->skip($start)
->take($stop)
->pluck('id');
@ -227,7 +241,7 @@ class NotificationService {
public static function getNotification($id)
{
$notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) {
$notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) {
$n = Notification::with('item')->find($id);
if(!$n) {
@ -259,7 +273,7 @@ class NotificationService {
public static function setNotification(Notification $notification)
{
return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) {
$fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
@ -270,8 +284,9 @@ class NotificationService {
public static function warmCache($id, $stop = 400, $force = false)
{
if(self::count($id) == 0 || $force == true) {
$ids = Notification::whereProfileId($id)
->latest()
$ids = Notification::where('profile_id', $id)
->where('id', '>', self::getEpochId())
->orderByDesc('id')
->limit($stop)
->pluck('id');
foreach($ids as $key) {

View file

@ -95,7 +95,7 @@ class PublicTimelineService {
if(self::count() == 0 || $force == true) {
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
Redis::del(self::CACHE_KEY);
$minId = SnowflakeService::byDate(now()->subDays(14));
$minId = SnowflakeService::byDate(now()->subDays(90));
$ids = Status::where('id', '>', $minId)
->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id'])
->when($hideNsfw, function($q, $hideNsfw) {
@ -105,9 +105,11 @@ class PublicTimelineService {
->whereScope('public')
->orderByDesc('id')
->limit($limit)
->pluck('id');
foreach($ids as $id) {
->pluck('id', 'profile_id');
foreach($ids as $k => $id) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) {
self::add($id);
}
}
return 1;
}

View file

@ -66,6 +66,14 @@ class RelationshipService
return self::get($aid, $tid);
}
public static function forget($aid, $tid)
{
Cache::forget('pf:services:follower:audience:' . $aid);
Cache::forget('pf:services:follower:audience:' . $tid);
self::delete($tid, $aid);
self::delete($aid, $tid);
}
public static function defaultRelation($tid)
{
return [

View file

@ -0,0 +1,66 @@
<?php
namespace App\Services;
use Storage;
use Illuminate\Http\File;
use Exception;
use GuzzleHttp\Exception\ClientException;
use Aws\S3\Exception\S3Exception;
use GuzzleHttp\Exception\ConnectException;
use League\Flysystem\UnableToWriteFile;
class ResilientMediaStorageService
{
static $attempts = 0;
public static function store($storagePath, $path, $name)
{
return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
self::handleResilientStore($storagePath, $path, $name) :
self::handleStore($storagePath, $path, $name);
}
public static function handleStore($storagePath, $path, $name)
{
return retry(3, function() use($storagePath, $path, $name) {
$baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
return $disk->url($file);
}, random_int(100, 500));
}
public static function handleResilientStore($storagePath, $path, $name)
{
$attempts = 0;
return retry(4, function() use($storagePath, $path, $name, $attempts) {
self::$attempts++;
usleep(100000);
$baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
try {
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
} catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
return $disk->url($file);
}, function (int $attempt, Exception $exception) {
return $attempt * 200;
});
}
public static function getAltDriver()
{
$drivers = [];
if(config('filesystems.disks.alt-primary.enabled')) {
$drivers[] = 'alt-primary';
}
if(config('filesystems.disks.alt-secondary.enabled')) {
$drivers[] = 'alt-secondary';
}
if(empty($drivers)) {
return false;
}
$key = array_rand($drivers, 1);
return $drivers[$key];
}
}

View file

@ -95,7 +95,15 @@ class SearchApiV2Service
if(substr($webfingerQuery, 0, 1) !== '@') {
$webfingerQuery = '@' . $webfingerQuery;
}
$banned = InstanceService::getBannedDomains();
$banned = InstanceService::getBannedDomains() ?? [];
$domainBlocks = UserFilterService::domainBlocks($user->profile_id);
if($domainBlocks && count($domainBlocks)) {
$banned = array_unique(
array_values(
array_merge($banned, $domainBlocks)
)
);
}
$operator = config('database.default') === 'pgsql' ? 'ilike' : 'like';
$results = Profile::select('username', 'id', 'followers_count', 'domain')
->where('username', $operator, $query)
@ -172,8 +180,18 @@ class SearchApiV2Service
'hashtags' => [],
'statuses' => [],
];
$user = request()->user();
$mastodonMode = self::$mastodonMode;
$query = urldecode($this->query->input('q'));
$banned = InstanceService::getBannedDomains();
$domainBlocks = UserFilterService::domainBlocks($user->profile_id);
if($domainBlocks && count($domainBlocks)) {
$banned = array_unique(
array_values(
array_merge($banned, $domainBlocks)
)
);
}
if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) {
$default['accounts'] = $this->accounts(substr($query, 1));
return $default;
@ -197,7 +215,11 @@ class SearchApiV2Service
} catch (\Exception $e) {
return $default;
}
if($res && isset($res['id'])) {
if($res && isset($res['id'], $res['url'])) {
$domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
if(in_array($domain, $banned)) {
return $default;
}
$default['accounts'][] = $res;
return $default;
} else {
@ -212,6 +234,10 @@ class SearchApiV2Service
return $default;
}
if($res && isset($res['id'])) {
$domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
if(in_array($domain, $banned)) {
return $default;
}
$default['accounts'][] = $res;
return $default;
} else {
@ -221,6 +247,9 @@ class SearchApiV2Service
if($sid = Status::whereUri($query)->first()) {
$s = StatusService::get($sid->id, false);
if(!$s) {
return $default;
}
if(in_array($s['visibility'], ['public', 'unlisted'])) {
$default['statuses'][] = $s;
return $default;
@ -229,7 +258,7 @@ class SearchApiV2Service
try {
$res = ActivityPubFetchService::get($query);
$banned = InstanceService::getBannedDomains();
if($res) {
$json = json_decode($res, true);

View file

@ -84,10 +84,7 @@ class StatusHashtagService {
public static function statusTags($statusId)
{
$key = 'pf:services:sh:id:' . $statusId;
return Cache::remember($key, 604800, function() use($statusId) {
$status = Status::find($statusId);
$status = Status::with('hashtags')->find($statusId);
if(!$status) {
return [];
}
@ -96,6 +93,5 @@ class StatusHashtagService {
$fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
return $fractal->createData($resource)->toArray();
});
}
}

Some files were not shown because too many files have changed in this diff Show more