Merge pull request #4992 from pixelfed/staging

Update SearchApiV2Service, use more efficient query
This commit is contained in:
daniel 2024-03-07 03:34:10 -07:00 committed by GitHub
commit 7c2ecd8706
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 88 additions and 78 deletions

View file

@ -5,6 +5,11 @@
### Updates ### Updates
- Update SoftwareUpdateService, add command to refresh latest versions ([632f2cb6](https://github.com/pixelfed/pixelfed/commit/632f2cb6)) - Update SoftwareUpdateService, add command to refresh latest versions ([632f2cb6](https://github.com/pixelfed/pixelfed/commit/632f2cb6))
- Update Post.vue, fix cache bug ([3a27e637](https://github.com/pixelfed/pixelfed/commit/3a27e637))
- Update StatusHashtagService, use more efficient cached count ([592c8412](https://github.com/pixelfed/pixelfed/commit/592c8412))
- Update DiscoverController, handle discover hashtag redirects ([18382e8a](https://github.com/pixelfed/pixelfed/commit/18382e8a))
- Update ApiV1Controller, use admin filter service ([94503a1c](https://github.com/pixelfed/pixelfed/commit/94503a1c))
- Update SearchApiV2Service, use more efficient query ([cee618e8](https://github.com/pixelfed/pixelfed/commit/cee618e8))
- ([](https://github.com/pixelfed/pixelfed/commit/)) - ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.13 (2024-03-05)](https://github.com/pixelfed/pixelfed/compare/v0.11.12...v0.11.13) ## [v0.11.13 (2024-03-05)](https://github.com/pixelfed/pixelfed/compare/v0.11.12...v0.11.13)

View file

@ -2,28 +2,26 @@
namespace App\Services; namespace App\Services;
use Cache; use App\Hashtag;
use Illuminate\Support\Facades\Redis; use App\Profile;
use App\{Hashtag, Profile, Status}; use App\Status;
use App\Transformer\Api\AccountTransformer; use App\Transformer\Api\AccountTransformer;
use App\Transformer\Api\StatusTransformer;
use League\Fractal;
use League\Fractal\Serializer\ArraySerializer;
use League\Fractal\Pagination\IlluminatePaginatorAdapter;
use App\Util\ActivityPub\Helpers; use App\Util\ActivityPub\Helpers;
use Illuminate\Support\Str; use Illuminate\Support\Str;
use App\Services\AccountService; use League\Fractal;
use App\Services\HashtagService; use League\Fractal\Serializer\ArraySerializer;
use App\Services\StatusService;
class SearchApiV2Service class SearchApiV2Service
{ {
private $query; private $query;
static $mastodonMode = false;
public static $mastodonMode = false;
public static function query($query, $mastodonMode = false) public static function query($query, $mastodonMode = false)
{ {
self::$mastodonMode = $mastodonMode; self::$mastodonMode = $mastodonMode;
return (new self)->run($query); return (new self)->run($query);
} }
@ -32,51 +30,51 @@ class SearchApiV2Service
$this->query = $query; $this->query = $query;
$q = urldecode($query->input('q')); $q = urldecode($query->input('q'));
if($query->has('resolve') && if ($query->has('resolve') &&
( Str::startsWith($q, 'https://') || (Str::startsWith($q, 'https://') ||
Str::substrCount($q, '@') >= 1) Str::substrCount($q, '@') >= 1)
) { ) {
return $this->resolveQuery(); return $this->resolveQuery();
} }
if($query->has('type')) { if ($query->has('type')) {
switch ($query->input('type')) { switch ($query->input('type')) {
case 'accounts': case 'accounts':
return [ return [
'accounts' => $this->accounts(), 'accounts' => $this->accounts(),
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
break; break;
case 'hashtags': case 'hashtags':
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => $this->hashtags(), 'hashtags' => $this->hashtags(),
'statuses' => [] 'statuses' => [],
]; ];
break; break;
case 'statuses': case 'statuses':
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => $this->statuses() 'statuses' => $this->statuses(),
]; ];
break; break;
} }
} }
if($query->has('account_id')) { if ($query->has('account_id')) {
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => $this->statusesById() 'statuses' => $this->statusesById(),
]; ];
} }
return [ return [
'accounts' => $this->accounts(), 'accounts' => $this->accounts(),
'hashtags' => $this->hashtags(), 'hashtags' => $this->hashtags(),
'statuses' => $this->statuses() 'statuses' => $this->statuses(),
]; ];
} }
@ -87,17 +85,17 @@ class SearchApiV2Service
$limit = $this->query->input('limit') ?? 20; $limit = $this->query->input('limit') ?? 20;
$offset = $this->query->input('offset') ?? 0; $offset = $this->query->input('offset') ?? 0;
$rawQuery = $initalQuery ? $initalQuery : $this->query->input('q'); $rawQuery = $initalQuery ? $initalQuery : $this->query->input('q');
$query = $rawQuery . '%'; $query = $rawQuery.'%';
$webfingerQuery = $query; $webfingerQuery = $query;
if(Str::substrCount($rawQuery, '@') == 1 && substr($rawQuery, 0, 1) !== '@') { if (Str::substrCount($rawQuery, '@') == 1 && substr($rawQuery, 0, 1) !== '@') {
$query = '@' . $query; $query = '@'.$query;
} }
if(substr($webfingerQuery, 0, 1) !== '@') { if (substr($webfingerQuery, 0, 1) !== '@') {
$webfingerQuery = '@' . $webfingerQuery; $webfingerQuery = '@'.$webfingerQuery;
} }
$banned = InstanceService::getBannedDomains() ?? []; $banned = InstanceService::getBannedDomains() ?? [];
$domainBlocks = UserFilterService::domainBlocks($user->profile_id); $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
if($domainBlocks && count($domainBlocks)) { if ($domainBlocks && count($domainBlocks)) {
$banned = array_unique( $banned = array_unique(
array_values( array_values(
array_merge($banned, $domainBlocks) array_merge($banned, $domainBlocks)
@ -112,15 +110,15 @@ class SearchApiV2Service
->offset($offset) ->offset($offset)
->limit($limit) ->limit($limit)
->get() ->get()
->filter(function($profile) use ($banned) { ->filter(function ($profile) use ($banned) {
return in_array($profile->domain, $banned) == false; return in_array($profile->domain, $banned) == false;
}) })
->map(function($res) use($mastodonMode) { ->map(function ($res) use ($mastodonMode) {
return $mastodonMode ? return $mastodonMode ?
AccountService::getMastodon($res['id']) : AccountService::getMastodon($res['id']) :
AccountService::get($res['id']); AccountService::get($res['id']);
}) })
->filter(function($account) { ->filter(function ($account) {
return $account && isset($account['id']); return $account && isset($account['id']);
}) })
->values(); ->values();
@ -134,31 +132,31 @@ class SearchApiV2Service
$q = $this->query->input('q'); $q = $this->query->input('q');
$limit = $this->query->input('limit') ?? 20; $limit = $this->query->input('limit') ?? 20;
$offset = $this->query->input('offset') ?? 0; $offset = $this->query->input('offset') ?? 0;
$query = Str::startsWith($q, '#') ? '%' . substr($q, 1) . '%' : '%' . $q . '%'; $query = Str::startsWith($q, '#') ? substr($q, 1).'%' : $q;
$operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like';
return Hashtag::where('name', $operator, $query) return Hashtag::where('name', $operator, $query)
->orWhere('slug', $operator, $query)
->where(function($q) {
return $q->where('can_search', true)
->orWhereNull('can_search');
})
->orderByDesc('cached_count') ->orderByDesc('cached_count')
->offset($offset) ->offset($offset)
->limit($limit) ->limit($limit)
->get() ->get()
->map(function($tag) use($mastodonMode) { ->filter(function ($tag) {
return $tag->can_search != false;
})
->map(function ($tag) use ($mastodonMode) {
$res = [ $res = [
'name' => $tag->name, 'name' => $tag->name,
'url' => $tag->url() 'url' => $tag->url(),
]; ];
if(!$mastodonMode) { if (! $mastodonMode) {
$res['history'] = []; $res['history'] = [];
$res['count'] = HashtagService::count($tag->id); $res['count'] = $tag->cached_count ?? 0;
} }
return $res; return $res;
}); })
->values();
} }
protected function statuses() protected function statuses()
@ -185,73 +183,77 @@ class SearchApiV2Service
$query = urldecode($this->query->input('q')); $query = urldecode($this->query->input('q'));
$banned = InstanceService::getBannedDomains(); $banned = InstanceService::getBannedDomains();
$domainBlocks = UserFilterService::domainBlocks($user->profile_id); $domainBlocks = UserFilterService::domainBlocks($user->profile_id);
if($domainBlocks && count($domainBlocks)) { if ($domainBlocks && count($domainBlocks)) {
$banned = array_unique( $banned = array_unique(
array_values( array_values(
array_merge($banned, $domainBlocks) array_merge($banned, $domainBlocks)
) )
); );
} }
if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) { if (substr($query, 0, 1) === '@' && ! Str::contains($query, '.')) {
$default['accounts'] = $this->accounts(substr($query, 1)); $default['accounts'] = $this->accounts(substr($query, 1));
return $default; return $default;
} }
if(Helpers::validateLocalUrl($query)) { if (Helpers::validateLocalUrl($query)) {
if(Str::contains($query, '/p/') || Str::contains($query, 'i/web/post/')) { if (Str::contains($query, '/p/') || Str::contains($query, 'i/web/post/')) {
return $this->resolveLocalStatus(); return $this->resolveLocalStatus();
} else if(Str::contains($query, 'i/web/profile/')) { } elseif (Str::contains($query, 'i/web/profile/')) {
return $this->resolveLocalProfileId(); return $this->resolveLocalProfileId();
} else { } else {
return $this->resolveLocalProfile(); return $this->resolveLocalProfile();
} }
} else { } else {
if(!Helpers::validateUrl($query) && strpos($query, '@') == -1) { if (! Helpers::validateUrl($query) && strpos($query, '@') == -1) {
return $default; return $default;
} }
if(!Str::startsWith($query, 'http') && Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) { if (! Str::startsWith($query, 'http') && Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) {
try { try {
$res = WebfingerService::lookup('@' . $query, $mastodonMode); $res = WebfingerService::lookup('@'.$query, $mastodonMode);
} catch (\Exception $e) { } catch (\Exception $e) {
return $default; return $default;
} }
if($res && isset($res['id'], $res['url'])) { if ($res && isset($res['id'], $res['url'])) {
$domain = strtolower(parse_url($res['url'], PHP_URL_HOST)); $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
if(in_array($domain, $banned)) { if (in_array($domain, $banned)) {
return $default; return $default;
} }
$default['accounts'][] = $res; $default['accounts'][] = $res;
return $default; return $default;
} else { } else {
return $default; return $default;
} }
} }
if(Str::substrCount($query, '@') == 2) { if (Str::substrCount($query, '@') == 2) {
try { try {
$res = WebfingerService::lookup($query, $mastodonMode); $res = WebfingerService::lookup($query, $mastodonMode);
} catch (\Exception $e) { } catch (\Exception $e) {
return $default; return $default;
} }
if($res && isset($res['id'])) { if ($res && isset($res['id'])) {
$domain = strtolower(parse_url($res['url'], PHP_URL_HOST)); $domain = strtolower(parse_url($res['url'], PHP_URL_HOST));
if(in_array($domain, $banned)) { if (in_array($domain, $banned)) {
return $default; return $default;
} }
$default['accounts'][] = $res; $default['accounts'][] = $res;
return $default; return $default;
} else { } else {
return $default; return $default;
} }
} }
if($sid = Status::whereUri($query)->first()) { if ($sid = Status::whereUri($query)->first()) {
$s = StatusService::get($sid->id, false); $s = StatusService::get($sid->id, false);
if(!$s) { if (! $s) {
return $default; return $default;
} }
if(in_array($s['visibility'], ['public', 'unlisted'])) { if (in_array($s['visibility'], ['public', 'unlisted'])) {
$default['statuses'][] = $s; $default['statuses'][] = $s;
return $default; return $default;
} }
} }
@ -259,10 +261,10 @@ class SearchApiV2Service
try { try {
$res = ActivityPubFetchService::get($query); $res = ActivityPubFetchService::get($query);
if($res) { if ($res) {
$json = json_decode($res, true); $json = json_decode($res, true);
if(!$json || !isset($json['@context']) || !isset($json['type']) || !in_array($json['type'], ['Note', 'Person'])) { if (! $json || ! isset($json['@context']) || ! isset($json['type']) || ! in_array($json['type'], ['Note', 'Person'])) {
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
@ -270,36 +272,38 @@ class SearchApiV2Service
]; ];
} }
switch($json['type']) { switch ($json['type']) {
case 'Note': case 'Note':
$obj = Helpers::statusFetch($query); $obj = Helpers::statusFetch($query);
if(!$obj || !isset($obj['id'])) { if (! $obj || ! isset($obj['id'])) {
return $default; return $default;
} }
$note = $mastodonMode ? $note = $mastodonMode ?
StatusService::getMastodon($obj['id'], false) : StatusService::getMastodon($obj['id'], false) :
StatusService::get($obj['id'], false); StatusService::get($obj['id'], false);
if(!$note) { if (! $note) {
return $default; return $default;
} }
if(!isset($note['visibility']) || !in_array($note['visibility'], ['public', 'unlisted'])) { if (! isset($note['visibility']) || ! in_array($note['visibility'], ['public', 'unlisted'])) {
return $default; return $default;
} }
$default['statuses'][] = $note; $default['statuses'][] = $note;
return $default; return $default;
break; break;
case 'Person': case 'Person':
$obj = Helpers::profileFetch($query); $obj = Helpers::profileFetch($query);
if(!$obj) { if (! $obj) {
return $default; return $default;
} }
if(in_array($obj['domain'], $banned)) { if (in_array($obj['domain'], $banned)) {
return $default; return $default;
} }
$default['accounts'][] = $mastodonMode ? $default['accounts'][] = $mastodonMode ?
AccountService::getMastodon($obj['id'], true) : AccountService::getMastodon($obj['id'], true) :
AccountService::get($obj['id'], true); AccountService::get($obj['id'], true);
return $default; return $default;
break; break;
@ -329,18 +333,18 @@ class SearchApiV2Service
$query = urldecode($this->query->input('q')); $query = urldecode($this->query->input('q'));
$query = last(explode('/', parse_url($query, PHP_URL_PATH))); $query = last(explode('/', parse_url($query, PHP_URL_PATH)));
$status = StatusService::getMastodon($query, false); $status = StatusService::getMastodon($query, false);
if(!$status || !in_array($status['visibility'], ['public', 'unlisted'])) { if (! $status || ! in_array($status['visibility'], ['public', 'unlisted'])) {
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
} }
$res = [ $res = [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => [$status] 'statuses' => [$status],
]; ];
return $res; return $res;
@ -355,21 +359,22 @@ class SearchApiV2Service
->whereUsername($query) ->whereUsername($query)
->first(); ->first();
if(!$profile) { if (! $profile) {
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
} }
$fractal = new Fractal\Manager(); $fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer()); $fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($profile, new AccountTransformer()); $resource = new Fractal\Resource\Item($profile, new AccountTransformer());
return [ return [
'accounts' => [$fractal->createData($resource)->toArray()], 'accounts' => [$fractal->createData($resource)->toArray()],
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
} }
@ -380,22 +385,22 @@ class SearchApiV2Service
$profile = Profile::whereNull('status') $profile = Profile::whereNull('status')
->find($query); ->find($query);
if(!$profile) { if (! $profile) {
return [ return [
'accounts' => [], 'accounts' => [],
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
} }
$fractal = new Fractal\Manager(); $fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer()); $fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($profile, new AccountTransformer()); $resource = new Fractal\Resource\Item($profile, new AccountTransformer());
return [ return [
'accounts' => [$fractal->createData($resource)->toArray()], 'accounts' => [$fractal->createData($resource)->toArray()],
'hashtags' => [], 'hashtags' => [],
'statuses' => [] 'statuses' => [],
]; ];
} }
} }