Add InstancePipeline and NodeinfoService

This commit is contained in:
Daniel Supernault 2021-09-03 20:45:56 -06:00
parent 7641b73158
commit da6943daed
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
4 changed files with 206 additions and 76 deletions

View file

@ -7,6 +7,9 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use App\Story; use App\Story;
use App\StoryView; use App\StoryView;
use App\Jobs\StoryPipeline\StoryExpire;
use App\Jobs\StoryPipeline\StoryRotateMedia;
use App\Services\StoryService;
class StoryGC extends Command class StoryGC extends Command
{ {
@ -41,89 +44,41 @@ class StoryGC extends Command
*/ */
public function handle() public function handle()
{ {
$this->directoryScan(); $this->archiveExpiredStories();
$this->deleteViews(); $this->rotateMedia();
$this->deleteStories();
} }
protected function directoryScan() protected function archiveExpiredStories()
{ {
$day = now()->day; $stories = Story::whereActive(true)
->where('expires_at', '<', now())
->get();
if($day !== 3) { foreach($stories as $story) {
StoryExpire::dispatch($story)->onQueue('story');
}
}
protected function rotateMedia()
{
$queue = StoryService::rotateQueue();
if(!$queue || count($queue) == 0) {
return; return;
} }
$monthHash = substr(hash('sha1', date('Y').date('m')), 0, 12); collect($queue)
->each(function($id) {
$t1 = Storage::directories('public/_esm.t1'); $story = StoryService::getById($id);
$t2 = Storage::directories('public/_esm.t2'); if(!$story) {
StoryService::removeRotateQueue($id);
$dirs = array_merge($t1, $t2); return;
}
foreach($dirs as $dir) { if($story->created_at->gt(now()->subMinutes(20))) {
$hash = last(explode('/', $dir)); return;
if($hash != $monthHash) { }
$this->info('Found directory to delete: ' . $dir); StoryRotateMedia::dispatch($story)->onQueue('story');
$this->deleteDirectory($dir); StoryService::removeRotateQueue($id);
}
}
$mh = hash('sha256', date('Y').'-.-'.date('m'));
$monthHash = date('Y').date('m').substr($mh, 0, 6).substr($mh, 58, 6);
$dirs = Storage::directories('public/_esm.t3');
foreach($dirs as $dir) {
$hash = last(explode('/', $dir));
if($hash != $monthHash) {
$this->info('Found directory to delete: ' . $dir);
$this->deleteDirectory($dir);
}
}
}
protected function deleteDirectory($path)
{
Storage::deleteDirectory($path);
}
protected function deleteViews()
{
StoryView::where('created_at', '<', now()->subMinutes(1441))->delete();
}
protected function deleteStories()
{
$stories = Story::where('created_at', '>', now()->subMinutes(30))
->whereNull('active')
->get();
foreach($stories as $story) {
if(Storage::exists($story->path) == true) {
Storage::delete($story->path);
}
DB::transaction(function() use($story) {
StoryView::whereStoryId($story->id)->delete();
$story->delete();
}); });
}
$stories = Story::where('created_at', '<', now()
->subMinutes(1441))
->get();
if($stories->count() == 0) {
exit;
}
foreach($stories as $story) {
if(Storage::exists($story->path) == true) {
Storage::delete($story->path);
}
DB::transaction(function() use($story) {
StoryView::whereStoryId($story->id)->delete();
$story->delete();
});
}
} }
} }

View file

@ -0,0 +1,56 @@
<?php
namespace App\Jobs\InstancePipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
use App\Instance;
use App\Profile;
use App\Services\NodeinfoService;
class FetchNodeinfoPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $instance;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Instance $instance)
{
$this->instance = $instance;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$instance = $this->instance;
$ni = NodeinfoService::get($instance->domain);
if($ni) {
if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) {
$software = $ni['software']['name'];
$instance->software = strtolower(strip_tags($software));
$instance->last_crawled_at = now();
$instance->user_count = Profile::whereDomain($instance->domain)->count();
$instance->save();
}
} else {
$instance->user_count = Profile::whereDomain($instance->domain)->count();
$instance->last_crawled_at = now();
$instance->save();
}
}
}

View file

@ -0,0 +1,43 @@
<?php
namespace App\Jobs\InstancePipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
use App\Instance;
use App\Profile;
use App\Services\NodeinfoService;
class InstanceCrawlPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct()
{
//
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
Instance::whereNull('last_crawled_at')->whereNull('software')->chunk(50, function($instances) use($headers) {
foreach($instances as $instance) {
FetchNodeinfoPipeline::dispatch($instance)->onQueue('low');
}
});
}
}

View file

@ -0,0 +1,76 @@
<?php
namespace App\Services;
use Illuminate\Support\Str;
use Illuminate\Support\Facades\Http;
use Illuminate\Http\Client\RequestException;
use Illuminate\Http\Client\ConnectionException;
class NodeinfoService
{
public static function get($domain)
{
$version = config('pixelfed.version');
$appUrl = config('app.url');
$headers = [
'Accept' => 'application/json',
'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})",
];
$url = 'https://' . $domain;
$wk = $url . '/.well-known/nodeinfo';
try {
$res = Http::withHeaders($headers)
->timeout(5)
->get($wk);
} catch (RequestException $e) {
return false;
} catch (ConnectionException $e) {
return false;
} catch (\Exception $e) {
return false;
}
if(!$res) {
return false;
}
$json = $res->json();
if( !isset($json['links'])) {
return false;
}
if(is_array($json['links'])) {
if(isset($json['links']['href'])) {
$href = $json['links']['href'];
} else {
$href = $json['links'][0]['href'];
}
} else {
return false;
}
$domain = parse_url($url, PHP_URL_HOST);
$hrefDomain = parse_url($href, PHP_URL_HOST);
if($domain !== $hrefDomain) {
return 60;
}
try {
$res = Http::withHeaders($headers)
->timeout(5)
->get($href);
} catch (RequestException $e) {
return false;
} catch (ConnectionException $e) {
return false;
} catch (\Exception $e) {
return false;
}
return $res->json();
}
}