mirror of
https://github.com/pixelfed/pixelfed.git
synced 2024-11-09 08:14:51 +00:00
Update HomeFeedPipeline, fix tag filtering
This commit is contained in:
Daniel Supernault
parent
e5401f8558
commit
f105f4e8f6
6 changed files with 267 additions and 242 deletions
|
|
@ -1,8 +1,8 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
indent_style = tab
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use App\Hashtag;
|
|||
use App\StatusHashtag;
|
||||
use App\Services\HashtagFollowService;
|
||||
use App\Services\HomeTimelineService;
|
||||
use App\Services\StatusService;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
|
||||
|
|
@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
|
|||
public function handle(): void
|
||||
{
|
||||
$hashtag = $this->hashtag;
|
||||
$sid = $hashtag->status_id;
|
||||
$status = StatusService::get($sid, false);
|
||||
|
||||
if(!$status) {
|
||||
return;
|
||||
}
|
||||
|
||||
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
|
||||
|
||||
if(!$ids || !count($ids)) {
|
||||
return;
|
||||
return;
|
||||
}
|
||||
|
||||
foreach($ids as $id) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use App\Hashtag;
|
|||
use App\StatusHashtag;
|
||||
use App\Services\HashtagFollowService;
|
||||
use App\Services\HomeTimelineService;
|
||||
use App\Services\StatusService;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
|
||||
|
|
@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
|
|||
{
|
||||
$sid = $this->sid;
|
||||
$hid = $this->hid;
|
||||
$status = StatusService::get($sid, false);
|
||||
|
||||
if(!$status) {
|
||||
return;
|
||||
}
|
||||
|
||||
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$ids = HashtagFollowService::getPidByHid($hid);
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
|
|||
{
|
||||
$hid = $this->hid;
|
||||
$pid = $this->pid;
|
||||
$slug = $this->slug;
|
||||
$slug = strtolower($this->slug);
|
||||
|
||||
$statusIds = HomeTimelineService::get($pid, 0, -1);
|
||||
|
||||
|
|
@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue
|
|||
|
||||
foreach($statusIds as $id) {
|
||||
$status = StatusService::get($id, false);
|
||||
if(!$status) {
|
||||
if(!$status || empty($status['tags'])) {
|
||||
HomeTimelineService::rem($pid, $id);
|
||||
continue;
|
||||
}
|
||||
$following = in_array($status['account']['id'], $followingIds);
|
||||
if($following || !isset($status['tags'])) {
|
||||
$following = in_array((int) $status['account']['id'], $followingIds);
|
||||
if($following === true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$tags = collect($status['tags'])->map(function($tag) {
|
||||
return $tag['name'];
|
||||
return strtolower($tag['name']);
|
||||
})->filter()->values()->toArray();
|
||||
|
||||
if(in_array($slug, $tags)) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
|
|||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use App\Services\StatusService;
|
||||
use App\Services\UserFilterService;
|
||||
use App\Services\AdminShadowFilterService;
|
||||
use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
|
||||
|
|
@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
|
|||
|
||||
class StatusEntityLexer implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $status;
|
||||
protected $entities;
|
||||
protected $autolink;
|
||||
protected $status;
|
||||
protected $entities;
|
||||
protected $autolink;
|
||||
|
||||
/**
|
||||
* Delete the job if its models no longer exist.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $deleteWhenMissingModels = true;
|
||||
/**
|
||||
* Delete the job if its models no longer exist.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(Status $status)
|
||||
{
|
||||
$this->status = $status;
|
||||
}
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(Status $status)
|
||||
{
|
||||
$this->status = $status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$profile = $this->status->profile;
|
||||
$status = $this->status;
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$profile = $this->status->profile;
|
||||
$status = $this->status;
|
||||
|
||||
if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
|
||||
$profile->status_count = $profile->status_count + 1;
|
||||
$profile->save();
|
||||
}
|
||||
if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
|
||||
$profile->status_count = $profile->status_count + 1;
|
||||
$profile->save();
|
||||
}
|
||||
|
||||
if($profile->no_autolink == false) {
|
||||
$this->parseEntities();
|
||||
}
|
||||
}
|
||||
if($profile->no_autolink == false) {
|
||||
$this->parseEntities();
|
||||
}
|
||||
}
|
||||
|
||||
public function parseEntities()
|
||||
{
|
||||
$this->extractEntities();
|
||||
}
|
||||
public function parseEntities()
|
||||
{
|
||||
$this->extractEntities();
|
||||
}
|
||||
|
||||
public function extractEntities()
|
||||
{
|
||||
$this->entities = Extractor::create()->extract($this->status->caption);
|
||||
$this->autolinkStatus();
|
||||
}
|
||||
public function extractEntities()
|
||||
{
|
||||
$this->entities = Extractor::create()->extract($this->status->caption);
|
||||
$this->autolinkStatus();
|
||||
}
|
||||
|
||||
public function autolinkStatus()
|
||||
{
|
||||
$this->autolink = Autolink::create()->autolink($this->status->caption);
|
||||
$this->storeEntities();
|
||||
}
|
||||
public function autolinkStatus()
|
||||
{
|
||||
$this->autolink = Autolink::create()->autolink($this->status->caption);
|
||||
$this->storeEntities();
|
||||
}
|
||||
|
||||
public function storeEntities()
|
||||
{
|
||||
$this->storeHashtags();
|
||||
DB::transaction(function () {
|
||||
$status = $this->status;
|
||||
$status->rendered = nl2br($this->autolink);
|
||||
$status->save();
|
||||
});
|
||||
}
|
||||
public function storeEntities()
|
||||
{
|
||||
$this->storeHashtags();
|
||||
DB::transaction(function () {
|
||||
$status = $this->status;
|
||||
$status->rendered = nl2br($this->autolink);
|
||||
$status->save();
|
||||
});
|
||||
}
|
||||
|
||||
public function storeHashtags()
|
||||
{
|
||||
$tags = array_unique($this->entities['hashtags']);
|
||||
$status = $this->status;
|
||||
public function storeHashtags()
|
||||
{
|
||||
$tags = array_unique($this->entities['hashtags']);
|
||||
$status = $this->status;
|
||||
|
||||
foreach ($tags as $tag) {
|
||||
if(mb_strlen($tag) > 124) {
|
||||
continue;
|
||||
}
|
||||
DB::transaction(function () use ($status, $tag) {
|
||||
$slug = str_slug($tag, '-', false);
|
||||
foreach ($tags as $tag) {
|
||||
if(mb_strlen($tag) > 124) {
|
||||
continue;
|
||||
}
|
||||
DB::transaction(function () use ($status, $tag) {
|
||||
$slug = str_slug($tag, '-', false);
|
||||
|
||||
$hashtag = Hashtag::firstOrCreate([
|
||||
'slug' => $slug
|
||||
|
|
@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue
|
|||
'name' => $tag
|
||||
]);
|
||||
|
||||
StatusHashtag::firstOrCreate(
|
||||
[
|
||||
'status_id' => $status->id,
|
||||
'hashtag_id' => $hashtag->id,
|
||||
'profile_id' => $status->profile_id,
|
||||
'status_visibility' => $status->visibility,
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
$this->storeMentions();
|
||||
}
|
||||
StatusHashtag::firstOrCreate(
|
||||
[
|
||||
'status_id' => $status->id,
|
||||
'hashtag_id' => $hashtag->id,
|
||||
'profile_id' => $status->profile_id,
|
||||
'status_visibility' => $status->visibility,
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
$this->storeMentions();
|
||||
}
|
||||
|
||||
public function storeMentions()
|
||||
{
|
||||
$mentions = array_unique($this->entities['mentions']);
|
||||
$status = $this->status;
|
||||
public function storeMentions()
|
||||
{
|
||||
$mentions = array_unique($this->entities['mentions']);
|
||||
$status = $this->status;
|
||||
|
||||
foreach ($mentions as $mention) {
|
||||
$mentioned = Profile::whereUsername($mention)->first();
|
||||
foreach ($mentions as $mention) {
|
||||
$mentioned = Profile::whereUsername($mention)->first();
|
||||
|
||||
if (empty($mentioned) || !isset($mentioned->id)) {
|
||||
continue;
|
||||
}
|
||||
if (empty($mentioned) || !isset($mentioned->id)) {
|
||||
continue;
|
||||
}
|
||||
$blocks = UserFilterService::blocks($mentioned->id);
|
||||
if($blocks && in_array($status->profile_id, $blocks)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DB::transaction(function () use ($status, $mentioned) {
|
||||
$m = new Mention();
|
||||
$m->status_id = $status->id;
|
||||
$m->profile_id = $mentioned->id;
|
||||
$m->save();
|
||||
DB::transaction(function () use ($status, $mentioned) {
|
||||
$m = new Mention();
|
||||
$m->status_id = $status->id;
|
||||
$m->profile_id = $mentioned->id;
|
||||
$m->save();
|
||||
|
||||
MentionPipeline::dispatch($status, $m);
|
||||
});
|
||||
}
|
||||
$this->fanout();
|
||||
}
|
||||
MentionPipeline::dispatch($status, $m);
|
||||
});
|
||||
}
|
||||
$this->fanout();
|
||||
}
|
||||
|
||||
public function fanout()
|
||||
{
|
||||
$status = $this->status;
|
||||
public function fanout()
|
||||
{
|
||||
$status = $this->status;
|
||||
StatusService::refresh($status->id);
|
||||
|
||||
if(config('exp.cached_home_timeline')) {
|
||||
if( $status->in_reply_to_id === null &&
|
||||
in_array($status->scope, ['public', 'unlisted', 'private'])
|
||||
) {
|
||||
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
|
||||
}
|
||||
}
|
||||
$this->deliver();
|
||||
}
|
||||
|
||||
public function deliver()
|
||||
{
|
||||
$status = $this->status;
|
||||
$types = [
|
||||
'photo',
|
||||
'photo:album',
|
||||
'video',
|
||||
'video:album',
|
||||
'photo:video:album'
|
||||
];
|
||||
|
||||
if(config_cache('pixelfed.bouncer.enabled')) {
|
||||
Bouncer::get($status);
|
||||
}
|
||||
|
||||
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
|
||||
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
|
||||
if( $status->uri == null &&
|
||||
$status->scope == 'public' &&
|
||||
in_array($status->type, $types) &&
|
||||
$status->in_reply_to_id === null &&
|
||||
$status->reblog_of_id === null &&
|
||||
($hideNsfw ? $status->is_nsfw == false : true)
|
||||
) {
|
||||
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
|
||||
PublicTimelineService::add($status->id);
|
||||
if(config('exp.cached_home_timeline')) {
|
||||
if( $status->in_reply_to_id === null &&
|
||||
in_array($status->scope, ['public', 'unlisted', 'private'])
|
||||
) {
|
||||
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->deliver();
|
||||
}
|
||||
|
||||
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
|
||||
StatusActivityPubDeliver::dispatch($status);
|
||||
}
|
||||
}
|
||||
public function deliver()
|
||||
{
|
||||
$status = $this->status;
|
||||
$types = [
|
||||
'photo',
|
||||
'photo:album',
|
||||
'video',
|
||||
'video:album',
|
||||
'photo:video:album'
|
||||
];
|
||||
|
||||
if(config_cache('pixelfed.bouncer.enabled')) {
|
||||
Bouncer::get($status);
|
||||
}
|
||||
|
||||
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
|
||||
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
|
||||
if( $status->uri == null &&
|
||||
$status->scope == 'public' &&
|
||||
in_array($status->type, $types) &&
|
||||
$status->in_reply_to_id === null &&
|
||||
$status->reblog_of_id === null &&
|
||||
($hideNsfw ? $status->is_nsfw == false : true)
|
||||
) {
|
||||
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
|
||||
PublicTimelineService::add($status->id);
|
||||
}
|
||||
}
|
||||
|
||||
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
|
||||
StatusActivityPubDeliver::dispatch($status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;
|
|||
|
||||
class StatusTagsPipeline implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $activity;
|
||||
protected $status;
|
||||
protected $activity;
|
||||
protected $status;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct($activity, $status)
|
||||
{
|
||||
$this->activity = $activity;
|
||||
$this->status = $status;
|
||||
}
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct($activity, $status)
|
||||
{
|
||||
$this->activity = $activity;
|
||||
$this->status = $status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$res = $this->activity;
|
||||
$status = $this->status;
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$res = $this->activity;
|
||||
$status = $this->status;
|
||||
|
||||
if(isset($res['tag']['type'], $res['tag']['name'])) {
|
||||
$res['tag'] = [$res['tag']];
|
||||
}
|
||||
|
||||
$tags = collect($res['tag']);
|
||||
$tags = collect($res['tag']);
|
||||
|
||||
// Emoji
|
||||
$tags->filter(function($tag) {
|
||||
return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
|
||||
})
|
||||
->map(function($tag) {
|
||||
CustomEmojiService::import($tag['id'], $this->status->id);
|
||||
});
|
||||
// Emoji
|
||||
$tags->filter(function($tag) {
|
||||
return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
|
||||
})
|
||||
->map(function($tag) {
|
||||
CustomEmojiService::import($tag['id'], $this->status->id);
|
||||
});
|
||||
|
||||
// Hashtags
|
||||
$tags->filter(function($tag) {
|
||||
return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
|
||||
})
|
||||
->map(function($tag) use($status) {
|
||||
$name = substr($tag['name'], 0, 1) == '#' ?
|
||||
substr($tag['name'], 1) : $tag['name'];
|
||||
// Hashtags
|
||||
$tags->filter(function($tag) {
|
||||
return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
|
||||
})
|
||||
->map(function($tag) use($status) {
|
||||
$name = substr($tag['name'], 0, 1) == '#' ?
|
||||
substr($tag['name'], 1) : $tag['name'];
|
||||
|
||||
$banned = TrendingHashtagService::getBannedHashtagNames();
|
||||
$banned = TrendingHashtagService::getBannedHashtagNames();
|
||||
|
||||
if(count($banned)) {
|
||||
if(count($banned)) {
|
||||
if(in_array(strtolower($name), array_map('strtolower', $banned))) {
|
||||
return;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if(config('database.default') === 'pgsql') {
|
||||
$hashtag = Hashtag::where('name', 'ilike', $name)
|
||||
->orWhere('slug', 'ilike', str_slug($name, '-', false))
|
||||
->first();
|
||||
$hashtag = Hashtag::where('name', 'ilike', $name)
|
||||
->orWhere('slug', 'ilike', str_slug($name, '-', false))
|
||||
->first();
|
||||
|
||||
if(!$hashtag) {
|
||||
$hashtag = Hashtag::updateOrCreate([
|
||||
'slug' => str_slug($name, '-', false),
|
||||
'name' => $name
|
||||
]);
|
||||
}
|
||||
if(!$hashtag) {
|
||||
$hashtag = Hashtag::updateOrCreate([
|
||||
'slug' => str_slug($name, '-', false),
|
||||
'name' => $name
|
||||
]);
|
||||
}
|
||||
} else {
|
||||
$hashtag = Hashtag::updateOrCreate([
|
||||
'slug' => str_slug($name, '-', false),
|
||||
'name' => $name
|
||||
]);
|
||||
$hashtag = Hashtag::updateOrCreate([
|
||||
'slug' => str_slug($name, '-', false),
|
||||
'name' => $name
|
||||
]);
|
||||
}
|
||||
|
||||
StatusHashtag::firstOrCreate([
|
||||
'status_id' => $status->id,
|
||||
'hashtag_id' => $hashtag->id,
|
||||
'profile_id' => $status->profile_id,
|
||||
'status_visibility' => $status->scope
|
||||
]);
|
||||
});
|
||||
StatusHashtag::firstOrCreate([
|
||||
'status_id' => $status->id,
|
||||
'hashtag_id' => $hashtag->id,
|
||||
'profile_id' => $status->profile_id,
|
||||
'status_visibility' => $status->scope
|
||||
]);
|
||||
});
|
||||
|
||||
// Mentions
|
||||
$tags->filter(function($tag) {
|
||||
return $tag &&
|
||||
$tag['type'] == 'Mention' &&
|
||||
isset($tag['href']) &&
|
||||
substr($tag['href'], 0, 8) === 'https://';
|
||||
})
|
||||
->map(function($tag) use($status) {
|
||||
if(Helpers::validateLocalUrl($tag['href'])) {
|
||||
$parts = explode('/', $tag['href']);
|
||||
if(!$parts) {
|
||||
return;
|
||||
}
|
||||
$pid = AccountService::usernameToId(end($parts));
|
||||
if(!$pid) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
$acct = Helpers::profileFetch($tag['href']);
|
||||
if(!$acct) {
|
||||
return;
|
||||
}
|
||||
$pid = $acct->id;
|
||||
}
|
||||
$mention = new Mention;
|
||||
$mention->status_id = $status->id;
|
||||
$mention->profile_id = $pid;
|
||||
$mention->save();
|
||||
MentionPipeline::dispatch($status, $mention);
|
||||
});
|
||||
}
|
||||
// Mentions
|
||||
$tags->filter(function($tag) {
|
||||
return $tag &&
|
||||
$tag['type'] == 'Mention' &&
|
||||
isset($tag['href']) &&
|
||||
substr($tag['href'], 0, 8) === 'https://';
|
||||
})
|
||||
->map(function($tag) use($status) {
|
||||
if(Helpers::validateLocalUrl($tag['href'])) {
|
||||
$parts = explode('/', $tag['href']);
|
||||
if(!$parts) {
|
||||
return;
|
||||
}
|
||||
$pid = AccountService::usernameToId(end($parts));
|
||||
if(!$pid) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
$acct = Helpers::profileFetch($tag['href']);
|
||||
if(!$acct) {
|
||||
return;
|
||||
}
|
||||
$pid = $acct->id;
|
||||
}
|
||||
$mention = new Mention;
|
||||
$mention->status_id = $status->id;
|
||||
$mention->profile_id = $pid;
|
||||
$mention->save();
|
||||
MentionPipeline::dispatch($status, $mention);
|
||||
});
|
||||
|
||||
StatusService::refresh($status->id);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue