YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2026-03-31 17:55:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix custom encoding conversion

Browse source
This commit is contained in:
yggverse 2024-03-21 03:38:46 +02:00
parent e09440b44a
commit 1f3ee435e9
1 changed files with 29 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/cli/document/crawl.php
View file

@ -201,14 +201,37 @@ foreach($index->search('')
} else continue; } else continue;
// Update MIME type or skip on empty // Update MIME type or skip on empty
if ($mime = curl_getinfo($request, CURLINFO_CONTENT_TYPE)) if ($type = curl_getinfo($request, CURLINFO_CONTENT_TYPE))
{ {
$data['mime'] = $mime; $data['mime'] = $type;
// On document charset specified
if (preg_match('/charset=([^\s;]+)/i', $type, $charset))
{
if (!empty($charset[1]))
{
// Get system encodings
foreach (mb_list_encodings() as $encoding)
{
if (strtolower($charset[1]) == strtolower($encoding))
{
// Convert response to UTF-8
$response = mb_convert_encoding(
$response,
'UTF-8',
$charset[1]
);
break;
}
}
}
}
} else continue; } else continue;
// DOM crawler // DOM crawler
if (false !== stripos($mime, 'text/html')) if (false !== stripos($type, 'text/html'))
{ {
$crawler = new Symfony\Component\DomCrawler\Crawler(); $crawler = new Symfony\Component\DomCrawler\Crawler();
$crawler->addHtmlContent( $crawler->addHtmlContent(
@ -579,7 +602,7 @@ foreach($index->search('')
$snap->addFromString( $snap->addFromString(
'MIME', 'MIME',
$mime $type
); );
$snap->addFromString( $snap->addFromString(
@ -608,7 +631,7 @@ foreach($index->search('')
foreach ($config->snap->storage->local->mime->stripos as $whitelist) foreach ($config->snap->storage->local->mime->stripos as $whitelist)
{ {
if (false !== stripos($mime, $whitelist)) if (false !== stripos($type, $whitelist))
{ {
$allowed = true; $allowed = true;
break; break;
@ -707,7 +730,7 @@ foreach($index->search('')
foreach ($ftp->mime->stripos as $whitelist) foreach ($ftp->mime->stripos as $whitelist)
{ {
if (false !== stripos($mime, $whitelist)) if (false !== stripos($type, $whitelist))
{ {
$allowed = true; $allowed = true;
break; break;