YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2026-03-31 17:55:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add crc32url filter

Browse source
This commit is contained in:
ghost 2023-11-25 18:10:23 +02:00
parent 8a827bfcdf
commit dfb2c06738
4 changed files with 44 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/cli/document/add.php
View file

@ -23,8 +23,13 @@ $index = $client->index(
$config->manticore->index->document->name
);
// Prepare URL
$url = trim($argv[1]);
$crc32url = crc32($url);
// Check URL for exist
$result = $index->search('@url "' . trim($argv[1]) . '"')
$result = $index->search('@url "' . $url . '"')
->filter('crc32url', $crc32url)
->limit(1)
->get();
@ -32,7 +37,7 @@ if ($result->getTotal())
{
echo sprintf(
'URL "%s" already exists in "%s" index!' . PHP_EOL,
$argv[1],
$url,
$config->manticore->index->document->name
);
@ -42,13 +47,14 @@ if ($result->getTotal())
// Add
$result = $index->addDocument(
[
'url' => trim($argv[1])
'url' => $url,
'crc32url' => $crc32url
]
);
echo sprintf(
'URL "%s" added to "%s" index: %s' . PHP_EOL,
$argv[1],
$url,
$config->manticore->index->document->name,
print_r(
$result,

5
src/cli/document/crawl.php
View file

@ -253,15 +253,18 @@ foreach($search->get() as $document)
foreach (array_unique($documents) as $url)
{
$url = trim($url);
$crc32url = crc32($url);
if (!$index->search('@url "' . $url . '"')
->filter('crc32url', $crc32url)
->limit(1)
->get()
->getTotal())
{
$index->addDocument(
[
'url' => $url
'url' => $url,
'crc32url' => $crc32url
]
);

4
src/cli/index/init.php
View file

@ -79,6 +79,10 @@ $result = $index->create(
'time' =>
[
'type' => 'integer'
],
'crc32url' =>
[
'type' => 'bigint'
]
],
(array) $config->manticore->index->document->settings

38
src/webui/search.php
View file

@ -69,7 +69,7 @@ $placeholder = plural(
$response = false;
// Request
$q = !empty($_GET['q']) ? $_GET['q'] : '';
$q = !empty($_GET['q']) ? trim($_GET['q']) : '';
$p = !empty($_GET['p']) ? (int) $_GET['p'] : 1;
// Register new URL by request on enabled
@ -77,10 +77,13 @@ if ($config->webui->search->index->request->url->enabled)
{
if (filter_var($q, FILTER_VALIDATE_URL) && preg_match($config->webui->search->index->request->url->regex, $q))
{
$url = trim($q);
// Prepare URL
$url = $q;
$crc32url = crc32($url);
// Check URL for exist
$exist = $index->search('@url "' . trim($url) . '"')
$exist = $index->search('@url "' . $url . '"')
->filter('crc32url', $crc32url)
->limit(1)
->get()
->getTotal();
@ -90,7 +93,7 @@ if ($config->webui->search->index->request->url->enabled)
/* disable as regular search request possible
$response = sprintf(
_('URL "%s" exists in search index'),
htmlentities($url)
htmlentities($q)
);
*/
}
@ -98,36 +101,39 @@ if ($config->webui->search->index->request->url->enabled)
// Add URL
else
{
// @TODO check http code
$index->addDocument(
[
'url' => trim($url)
'url' => $url,
'crc32url' => $crc32url
]
);
$response = sprintf(
_('URL "%s" added to the crawl queue!'),
htmlentities($url)
htmlentities($q)
);
}
}
}
// Extended syntax corrections
$query = trim($q);
if (filter_var($q, FILTER_VALIDATE_URL))
switch (true)
{
$query = '@url "' . $q . '"';
}
case filter_var($q, FILTER_VALIDATE_URL):
elseif (false === strpos($q, '"'))
{
$query = '"' . $q . '"';
$query = $index->search('@url "' . $q . '"')->filter('crc32url', crc32($q));
break;
default:
$query = $index->search($q);
}
// Search request begin
$results = $index->search($query)
->offset($p * $config->webui->pagination->limit - $config->webui->pagination->limit)
$results = $query->offset($p * $config->webui->pagination->limit - $config->webui->pagination->limit)
->limit($config->webui->pagination->limit)
->get();