UA-Fediland/pixelfed
2
0
Fork 0
mirror of https://github.com/pixelfed/pixelfed.git synced 2024-11-22 06:21:27 +00:00
Code Issues Projects Releases Packages Wiki Activity

Update HomeFeedPipeline, fix tag filtering

Browse source
This commit is contained in:
Daniel Supernault 2023-11-16 00:47:49 -07:00
parent e5401f8558
commit f105f4e8f6
No known key found for this signature in database
GPG key ID: 23740873EE6F76A1
6 changed files with 267 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.editorconfig
View file

@ -1,8 +1,8 @@
root = true root = true
[*] [*]
indent_style = space
indent_size = 4 indent_size = 4
indent_style = tab
end_of_line = lf end_of_line = lf
charset = utf-8 charset = utf-8
trim_trailing_whitespace = true trim_trailing_whitespace = true

13
app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php
View file

@ -12,6 +12,7 @@ use App\Hashtag;
use App\StatusHashtag; use App\StatusHashtag;
use App\Services\HashtagFollowService; use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService; use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
public function handle(): void public function handle(): void
{ {
$hashtag = $this->hashtag; $hashtag = $this->hashtag;
$sid = $hashtag->status_id;
$status = StatusService::get($sid, false);
if(!$status) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id); $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
if(!$ids || !count($ids)) { if(!$ids || !count($ids)) {
return; return;
} }
foreach($ids as $id) { foreach($ids as $id) {

10
app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php
View file

@ -12,6 +12,7 @@ use App\Hashtag;
use App\StatusHashtag; use App\StatusHashtag;
use App\Services\HashtagFollowService; use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService; use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
{ {
$sid = $this->sid; $sid = $this->sid;
$hid = $this->hid; $hid = $this->hid;
$status = StatusService::get($sid, false);
if(!$status) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = HashtagFollowService::getPidByHid($hid); $ids = HashtagFollowService::getPidByHid($hid);

10
app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php
View file

@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
{ {
$hid = $this->hid; $hid = $this->hid;
$pid = $this->pid; $pid = $this->pid;
$slug = $this->slug; $slug = strtolower($this->slug);
$statusIds = HomeTimelineService::get($pid, 0, -1); $statusIds = HomeTimelineService::get($pid, 0, -1);
@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue
foreach($statusIds as $id) { foreach($statusIds as $id) {
$status = StatusService::get($id, false); $status = StatusService::get($id, false);
if(!$status) { if(!$status || empty($status['tags'])) {
HomeTimelineService::rem($pid, $id); HomeTimelineService::rem($pid, $id);
continue; continue;
} }
$following = in_array($status['account']['id'], $followingIds); $following = in_array((int) $status['account']['id'], $followingIds);
if($following || !isset($status['tags'])) { if($following === true) {
continue; continue;
} }
$tags = collect($status['tags'])->map(function($tag) { $tags = collect($status['tags'])->map(function($tag) {
return $tag['name']; return strtolower($tag['name']);
})->filter()->values()->toArray(); })->filter()->values()->toArray();
if(in_array($slug, $tags)) { if(in_array($slug, $tags)) {

290
app/Jobs/StatusPipeline/StatusEntityLexer.php
View file

@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use App\Services\StatusService;
use App\Services\UserFilterService; use App\Services\UserFilterService;
use App\Services\AdminShadowFilterService; use App\Services\AdminShadowFilterService;
use App\Jobs\HomeFeedPipeline\FeedInsertPipeline; use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
class StatusEntityLexer implements ShouldQueue class StatusEntityLexer implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $status; protected $status;
protected $entities; protected $entities;
protected $autolink; protected $autolink;
/** /**
* Delete the job if its models no longer exist. * Delete the job if its models no longer exist.
* *
* @var bool * @var bool
*/ */
public $deleteWhenMissingModels = true; public $deleteWhenMissingModels = true;
/** /**
* Create a new job instance. * Create a new job instance.
* *
* @return void * @return void
*/ */
public function __construct(Status $status) public function __construct(Status $status)
{ {
$this->status = $status; $this->status = $status;
} }
/** /**
* Execute the job. * Execute the job.
* *
* @return void * @return void
*/ */
public function handle() public function handle()
{ {
$profile = $this->status->profile; $profile = $this->status->profile;
$status = $this->status; $status = $this->status;
if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) { if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
$profile->status_count = $profile->status_count + 1; $profile->status_count = $profile->status_count + 1;
$profile->save(); $profile->save();
} }
if($profile->no_autolink == false) { if($profile->no_autolink == false) {
$this->parseEntities(); $this->parseEntities();
} }
} }
public function parseEntities() public function parseEntities()
{ {
$this->extractEntities(); $this->extractEntities();
} }
public function extractEntities() public function extractEntities()
{ {
$this->entities = Extractor::create()->extract($this->status->caption); $this->entities = Extractor::create()->extract($this->status->caption);
$this->autolinkStatus(); $this->autolinkStatus();
} }
public function autolinkStatus() public function autolinkStatus()
{ {
$this->autolink = Autolink::create()->autolink($this->status->caption); $this->autolink = Autolink::create()->autolink($this->status->caption);
$this->storeEntities(); $this->storeEntities();
} }
public function storeEntities() public function storeEntities()
{ {
$this->storeHashtags(); $this->storeHashtags();
DB::transaction(function () { DB::transaction(function () {
$status = $this->status; $status = $this->status;
$status->rendered = nl2br($this->autolink); $status->rendered = nl2br($this->autolink);
$status->save(); $status->save();
}); });
} }
public function storeHashtags() public function storeHashtags()
{ {
$tags = array_unique($this->entities['hashtags']); $tags = array_unique($this->entities['hashtags']);
$status = $this->status; $status = $this->status;
foreach ($tags as $tag) { foreach ($tags as $tag) {
if(mb_strlen($tag) > 124) { if(mb_strlen($tag) > 124) {
continue; continue;
} }
DB::transaction(function () use ($status, $tag) { DB::transaction(function () use ($status, $tag) {
$slug = str_slug($tag, '-', false); $slug = str_slug($tag, '-', false);
$hashtag = Hashtag::firstOrCreate([ $hashtag = Hashtag::firstOrCreate([
'slug' => $slug 'slug' => $slug
@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue
'name' => $tag 'name' => $tag
]); ]);
StatusHashtag::firstOrCreate( StatusHashtag::firstOrCreate(
[ [
'status_id' => $status->id, 'status_id' => $status->id,
'hashtag_id' => $hashtag->id, 'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id, 'profile_id' => $status->profile_id,
'status_visibility' => $status->visibility, 'status_visibility' => $status->visibility,
] ]
); );
}); });
} }
$this->storeMentions(); $this->storeMentions();
} }
public function storeMentions() public function storeMentions()
{ {
$mentions = array_unique($this->entities['mentions']); $mentions = array_unique($this->entities['mentions']);
$status = $this->status; $status = $this->status;
foreach ($mentions as $mention) { foreach ($mentions as $mention) {
$mentioned = Profile::whereUsername($mention)->first(); $mentioned = Profile::whereUsername($mention)->first();
if (empty($mentioned) || !isset($mentioned->id)) { if (empty($mentioned) || !isset($mentioned->id)) {
continue; continue;
} }
$blocks = UserFilterService::blocks($mentioned->id); $blocks = UserFilterService::blocks($mentioned->id);
if($blocks && in_array($status->profile_id, $blocks)) { if($blocks && in_array($status->profile_id, $blocks)) {
continue; continue;
} }
DB::transaction(function () use ($status, $mentioned) { DB::transaction(function () use ($status, $mentioned) {
$m = new Mention(); $m = new Mention();
$m->status_id = $status->id; $m->status_id = $status->id;
$m->profile_id = $mentioned->id; $m->profile_id = $mentioned->id;
$m->save(); $m->save();
MentionPipeline::dispatch($status, $m); MentionPipeline::dispatch($status, $m);
}); });
} }
$this->fanout(); $this->fanout();
} }
public function fanout() public function fanout()
{ {
$status = $this->status; $status = $this->status;
StatusService::refresh($status->id);
if(config('exp.cached_home_timeline')) { if(config('exp.cached_home_timeline')) {
if( $status->in_reply_to_id === null && if( $status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private']) in_array($status->scope, ['public', 'unlisted', 'private'])
) { ) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed'); FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}
$this->deliver();
}
public function deliver()
{
$status = $this->status;
$types = [
'photo',
'photo:album',
'video',
'video:album',
'photo:video:album'
];
if(config_cache('pixelfed.bouncer.enabled')) {
Bouncer::get($status);
}
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
if( $status->uri == null &&
$status->scope == 'public' &&
in_array($status->type, $types) &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
($hideNsfw ? $status->is_nsfw == false : true)
) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
PublicTimelineService::add($status->id);
} }
} }
$this->deliver();
}
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') { public function deliver()
StatusActivityPubDeliver::dispatch($status); {
} $status = $this->status;
} $types = [
'photo',
'photo:album',
'video',
'video:album',
'photo:video:album'
];
if(config_cache('pixelfed.bouncer.enabled')) {
Bouncer::get($status);
}
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
if( $status->uri == null &&
$status->scope == 'public' &&
in_array($status->type, $types) &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
($hideNsfw ? $status->is_nsfw == false : true)
) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
PublicTimelineService::add($status->id);
}
}
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
StatusActivityPubDeliver::dispatch($status);
}
}
} }

184
app/Jobs/StatusPipeline/StatusTagsPipeline.php
View file

@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;
class StatusTagsPipeline implements ShouldQueue class StatusTagsPipeline implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $activity; protected $activity;
protected $status; protected $status;
/** /**
* Create a new job instance. * Create a new job instance.
* *
* @return void * @return void
*/ */
public function __construct($activity, $status) public function __construct($activity, $status)
{ {
$this->activity = $activity; $this->activity = $activity;
$this->status = $status; $this->status = $status;
} }
/** /**
* Execute the job. * Execute the job.
* *
* @return void * @return void
*/ */
public function handle() public function handle()
{ {
$res = $this->activity; $res = $this->activity;
$status = $this->status; $status = $this->status;
if(isset($res['tag']['type'], $res['tag']['name'])) { if(isset($res['tag']['type'], $res['tag']['name'])) {
$res['tag'] = [$res['tag']]; $res['tag'] = [$res['tag']];
} }
$tags = collect($res['tag']); $tags = collect($res['tag']);
// Emoji // Emoji
$tags->filter(function($tag) { $tags->filter(function($tag) {
return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji'; return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
}) })
->map(function($tag) { ->map(function($tag) {
CustomEmojiService::import($tag['id'], $this->status->id); CustomEmojiService::import($tag['id'], $this->status->id);
}); });
// Hashtags // Hashtags
$tags->filter(function($tag) { $tags->filter(function($tag) {
return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']); return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
}) })
->map(function($tag) use($status) { ->map(function($tag) use($status) {
$name = substr($tag['name'], 0, 1) == '#' ? $name = substr($tag['name'], 0, 1) == '#' ?
substr($tag['name'], 1) : $tag['name']; substr($tag['name'], 1) : $tag['name'];
$banned = TrendingHashtagService::getBannedHashtagNames(); $banned = TrendingHashtagService::getBannedHashtagNames();
if(count($banned)) { if(count($banned)) {
if(in_array(strtolower($name), array_map('strtolower', $banned))) { if(in_array(strtolower($name), array_map('strtolower', $banned))) {
return; return;
} }
} }
if(config('database.default') === 'pgsql') { if(config('database.default') === 'pgsql') {
$hashtag = Hashtag::where('name', 'ilike', $name) $hashtag = Hashtag::where('name', 'ilike', $name)
->orWhere('slug', 'ilike', str_slug($name, '-', false)) ->orWhere('slug', 'ilike', str_slug($name, '-', false))
->first(); ->first();
if(!$hashtag) { if(!$hashtag) {
$hashtag = Hashtag::updateOrCreate([ $hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false), 'slug' => str_slug($name, '-', false),
'name' => $name 'name' => $name
]); ]);
} }
} else { } else {
$hashtag = Hashtag::updateOrCreate([ $hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false), 'slug' => str_slug($name, '-', false),
'name' => $name 'name' => $name
]); ]);
} }
StatusHashtag::firstOrCreate([ StatusHashtag::firstOrCreate([
'status_id' => $status->id, 'status_id' => $status->id,
'hashtag_id' => $hashtag->id, 'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id, 'profile_id' => $status->profile_id,
'status_visibility' => $status->scope 'status_visibility' => $status->scope
]); ]);
}); });
// Mentions // Mentions
$tags->filter(function($tag) { $tags->filter(function($tag) {
return $tag && return $tag &&
$tag['type'] == 'Mention' && $tag['type'] == 'Mention' &&
isset($tag['href']) && isset($tag['href']) &&
substr($tag['href'], 0, 8) === 'https://'; substr($tag['href'], 0, 8) === 'https://';
}) })
->map(function($tag) use($status) { ->map(function($tag) use($status) {
if(Helpers::validateLocalUrl($tag['href'])) { if(Helpers::validateLocalUrl($tag['href'])) {
$parts = explode('/', $tag['href']); $parts = explode('/', $tag['href']);
if(!$parts) { if(!$parts) {
return; return;
} }
$pid = AccountService::usernameToId(end($parts)); $pid = AccountService::usernameToId(end($parts));
if(!$pid) { if(!$pid) {
return; return;
} }
} else { } else {
$acct = Helpers::profileFetch($tag['href']); $acct = Helpers::profileFetch($tag['href']);
if(!$acct) { if(!$acct) {
return; return;
} }
$pid = $acct->id; $pid = $acct->id;
} }
$mention = new Mention; $mention = new Mention;
$mention->status_id = $status->id; $mention->status_id = $status->id;
$mention->profile_id = $pid; $mention->profile_id = $pid;
$mention->save(); $mention->save();
MentionPipeline::dispatch($status, $mention); MentionPipeline::dispatch($status, $mention);
}); });
}
StatusService::refresh($status->id);
}
} }