YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2026-03-31 17:55:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

implement FTP snaps

Browse source
This commit is contained in:
ghost 2023-11-25 03:19:54 +02:00
parent f7d11079a8
commit 875382c56e
3 changed files with 219 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

3
composer.json
View file

@ -17,6 +17,7 @@
"manticoresoftware/manticoresearch-php": "^3.1", "manticoresoftware/manticoresearch-php": "^3.1",
"symfony/css-selector": "^6.3", "symfony/css-selector": "^6.3",
"symfony/dom-crawler": "^6.3", "symfony/dom-crawler": "^6.3",
"jdenticon/jdenticon": "^1.0" "jdenticon/jdenticon": "^1.0",
"yggverse/ftp": "^1.0"
} }
} }

44
example/config.json
View file

@ -15,7 +15,7 @@
{ {
"url": "url":
{ {
"base":"http://127.0.0.1:8888" "base":"http://127.0.0.1"
}, },
"pagination": "pagination":
{ {
@ -77,12 +77,15 @@
{ {
"storage": "storage":
{ {
"tmp":{
"directory":"storage/tmp/snap"
},
"local":{ "local":{
"enabled":true, "enabled":true,
"directory":"storage/snap", "directory":"storage/snap",
"size": "size":
{ {
"max":100024 "max":10000024
}, },
"mime": "mime":
[ [
@ -98,9 +101,44 @@
] ]
}, },
"mirror": "mirror":
{
"ftp":
[
{ {
"enabled":false, "enabled":false,
"ftp":[] "connection":
{
"port":21,
"host":"",
"username":"",
"password":"",
"directory":"/snap/yo",
"timeout":30,
"passive":true,
"attempts":
{
"limit":0,
"delay":60
}
},
"size":
{
"max":10000024
},
"mime":
[
"application/xhtml+xml",
"application/javascript",
"text/html",
"text/plain",
"text/css",
"image/webp",
"image/png",
"image/gif",
"image/ico"
]
}
]
} }
} }
} }

249
src/cli/document/crawl.php
View file

@ -287,6 +287,92 @@ foreach($search->get() as $document)
$document->get('url') $document->get('url')
); );
/// absolute
if ('/' === substr($config->snap->storage->tmp->directory, 0, 1))
{
$filepath = $config->snap->storage->tmp->directory;
}
/// relative
else
{
$filepath = __DIR__ . '/../../../' . $config->snap->storage->tmp->directory;
}
$filepath = sprintf(
'%s/%s',
$filepath,
implode(
'/',
str_split(
$md5url
)
)
);
@mkdir($filepath, 0755, true);
$tmp = sprintf(
'%s/%s.tar',
$filepath,
$time
);
// Compress response to archive
$snap = new PharData($tmp);
$snap->addFromString(
'DATA',
$response
);
$snap->addFromString(
'MIME',
$mime
);
$snap->addFromString(
'URL',
$document->get('url')
);
$snap->compress(
Phar::GZ
);
unlink( // remove tarball
$tmp
);
$tmp = sprintf(
'%s.gz',
$tmp
);
// Copy to local storage on enabled
if ($config->snap->storage->local->enabled)
{
$allowed = false;
// Check for mime allowed
foreach ($config->snap->storage->local->mime as $whitelist)
{
if (false !== stripos($mime, $whitelist))
{
$allowed = true;
break;
}
}
// Check size limits
if ($size > $config->snap->storage->local->size->max)
{
$allowed = false;
}
// Copy snap to the permanent storage
if ($allowed)
{
/// absolute /// absolute
if ('/' === substr($config->snap->storage->local->directory, 0, 1)) if ('/' === substr($config->snap->storage->local->directory, 0, 1))
{ {
@ -310,85 +396,132 @@ foreach($search->get() as $document)
) )
); );
$filename = sprintf(
'%s/%s.tar',
$filepath,
$time
);
@mkdir($filepath, 0755, true); @mkdir($filepath, 0755, true);
// Compress response to archive
$snap = new PharData($filename);
$snap->addFromString(
'DATA',
$response
);
$snap->addFromString(
'MIME',
$mime
);
$snap->addFromString(
'URL',
$document->get('url')
);
$snap->compress(
Phar::GZ
);
unlink(
$filename
);
$filename = sprintf( $filename = sprintf(
'%s.gz', '%s/%s',
$filename $filepath,
basename(
$tmp
)
); );
// Copy to mirror storage on enabled copy(
if ($config->snap->storage->mirror->enabled) $tmp,
{
// @TODO copy
// Snap match remote storage size/mime conditions
}
// Remove snap on local storage disabled
if (!$config->snap->storage->local->enabled)
{
@unlink(
$filename $filename
); );
} }
// Remove snap on out of local storage size limits
if ($size > $config->snap->storage->local->size->max)
{
@unlink(
$filename
);
} }
// Remove snap on mime not allowed // Copy to FTP mirror storage on enabled
$remove = true; foreach ($config->snap->storage->mirror->ftp as $ftp)
foreach ($config->snap->storage->local->mime as $whitelist) {
// Resource enabled
if (!$ftp->enabled)
{
continue;
}
$allowed = false;
// Check for mime allowed
foreach ($ftp->mime as $whitelist)
{ {
if (false !== stripos($mime, $whitelist)) if (false !== stripos($mime, $whitelist))
{ {
$remove = false; $allowed = true;
break; break;
} }
} }
if ($remove) // Check size limits
if ($size > $ftp->size->max)
{ {
@unlink( $allowed = false;
}
if (!$allowed)
{
continue;
}
// Prepare location
$filepath = implode(
'/',
str_split(
$md5url
)
);
$filename = sprintf(
'%s/%s',
$filepath,
basename(
$tmp
)
);
// Init connection
$attempt = 1;
do {
$remote = new \Yggverse\Ftp\Client();
$connection = $remote->connect(
$ftp->connection->host,
$ftp->connection->port,
$ftp->connection->username,
$ftp->connection->password,
$ftp->connection->directory,
$ftp->connection->timeout,
$ftp->connection->passive
);
// Remote host connected
if ($connection) {
$remote->mkdir(
$filepath,
true
);
$remote->copy(
$tmp,
$filename $filename
); );
$remote->close();
// On remote connection lost, repeat attempt
} else {
// Stop connection attempts on limit provided
if ($ftp->connection->attempts->limit > 0 && $attempt > $ftp->connection->attempts->limit)
{
break;
} }
// Log event
echo sprintf(
_('[attempt: %s] wait for remote storage "%s" reconnection...') . PHP_EOL,
$attempt++,
$ftp->connection->host,
);
// Delay next attempt
sleep(
$ftp->connection->attempts->delay
);
}
} while ($connection === false);
}
// Remove tmp data
@unlink(
$tmp
);
} }
catch (Exception $exception) catch (Exception $exception)