mirror of
https://github.com/pixelfed/pixelfed.git
synced 2024-11-10 00:34:50 +00:00
Add InstancePipeline and NodeinfoService
This commit is contained in:
parent
7641b73158
commit
da6943daed
4 changed files with 206 additions and 76 deletions
|
@ -7,6 +7,9 @@ use Illuminate\Support\Facades\DB;
|
|||
use Illuminate\Support\Facades\Storage;
|
||||
use App\Story;
|
||||
use App\StoryView;
|
||||
use App\Jobs\StoryPipeline\StoryExpire;
|
||||
use App\Jobs\StoryPipeline\StoryRotateMedia;
|
||||
use App\Services\StoryService;
|
||||
|
||||
class StoryGC extends Command
|
||||
{
|
||||
|
@ -41,89 +44,41 @@ class StoryGC extends Command
|
|||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->directoryScan();
|
||||
$this->deleteViews();
|
||||
$this->deleteStories();
|
||||
$this->archiveExpiredStories();
|
||||
$this->rotateMedia();
|
||||
}
|
||||
|
||||
protected function directoryScan()
|
||||
protected function archiveExpiredStories()
|
||||
{
|
||||
$day = now()->day;
|
||||
$stories = Story::whereActive(true)
|
||||
->where('expires_at', '<', now())
|
||||
->get();
|
||||
|
||||
if($day !== 3) {
|
||||
foreach($stories as $story) {
|
||||
StoryExpire::dispatch($story)->onQueue('story');
|
||||
}
|
||||
}
|
||||
|
||||
protected function rotateMedia()
|
||||
{
|
||||
$queue = StoryService::rotateQueue();
|
||||
|
||||
if(!$queue || count($queue) == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
$monthHash = substr(hash('sha1', date('Y').date('m')), 0, 12);
|
||||
|
||||
$t1 = Storage::directories('public/_esm.t1');
|
||||
$t2 = Storage::directories('public/_esm.t2');
|
||||
|
||||
$dirs = array_merge($t1, $t2);
|
||||
|
||||
foreach($dirs as $dir) {
|
||||
$hash = last(explode('/', $dir));
|
||||
if($hash != $monthHash) {
|
||||
$this->info('Found directory to delete: ' . $dir);
|
||||
$this->deleteDirectory($dir);
|
||||
collect($queue)
|
||||
->each(function($id) {
|
||||
$story = StoryService::getById($id);
|
||||
if(!$story) {
|
||||
StoryService::removeRotateQueue($id);
|
||||
return;
|
||||
}
|
||||
if($story->created_at->gt(now()->subMinutes(20))) {
|
||||
return;
|
||||
}
|
||||
|
||||
$mh = hash('sha256', date('Y').'-.-'.date('m'));
|
||||
$monthHash = date('Y').date('m').substr($mh, 0, 6).substr($mh, 58, 6);
|
||||
$dirs = Storage::directories('public/_esm.t3');
|
||||
|
||||
foreach($dirs as $dir) {
|
||||
$hash = last(explode('/', $dir));
|
||||
if($hash != $monthHash) {
|
||||
$this->info('Found directory to delete: ' . $dir);
|
||||
$this->deleteDirectory($dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function deleteDirectory($path)
|
||||
{
|
||||
Storage::deleteDirectory($path);
|
||||
}
|
||||
|
||||
protected function deleteViews()
|
||||
{
|
||||
StoryView::where('created_at', '<', now()->subMinutes(1441))->delete();
|
||||
}
|
||||
|
||||
protected function deleteStories()
|
||||
{
|
||||
$stories = Story::where('created_at', '>', now()->subMinutes(30))
|
||||
->whereNull('active')
|
||||
->get();
|
||||
|
||||
foreach($stories as $story) {
|
||||
if(Storage::exists($story->path) == true) {
|
||||
Storage::delete($story->path);
|
||||
}
|
||||
DB::transaction(function() use($story) {
|
||||
StoryView::whereStoryId($story->id)->delete();
|
||||
$story->delete();
|
||||
StoryRotateMedia::dispatch($story)->onQueue('story');
|
||||
StoryService::removeRotateQueue($id);
|
||||
});
|
||||
}
|
||||
|
||||
$stories = Story::where('created_at', '<', now()
|
||||
->subMinutes(1441))
|
||||
->get();
|
||||
|
||||
if($stories->count() == 0) {
|
||||
exit;
|
||||
}
|
||||
|
||||
foreach($stories as $story) {
|
||||
if(Storage::exists($story->path) == true) {
|
||||
Storage::delete($story->path);
|
||||
}
|
||||
DB::transaction(function() use($story) {
|
||||
StoryView::whereStoryId($story->id)->delete();
|
||||
$story->delete();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
56
app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
Normal file
56
app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
Normal file
|
@ -0,0 +1,56 @@
|
|||
<?php
|
||||
|
||||
namespace App\Jobs\InstancePipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use App\Instance;
|
||||
use App\Profile;
|
||||
use App\Services\NodeinfoService;
|
||||
|
||||
class FetchNodeinfoPipeline implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $instance;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(Instance $instance)
|
||||
{
|
||||
$this->instance = $instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$instance = $this->instance;
|
||||
|
||||
$ni = NodeinfoService::get($instance->domain);
|
||||
if($ni) {
|
||||
if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) {
|
||||
$software = $ni['software']['name'];
|
||||
$instance->software = strtolower(strip_tags($software));
|
||||
$instance->last_crawled_at = now();
|
||||
$instance->user_count = Profile::whereDomain($instance->domain)->count();
|
||||
$instance->save();
|
||||
}
|
||||
} else {
|
||||
$instance->user_count = Profile::whereDomain($instance->domain)->count();
|
||||
$instance->last_crawled_at = now();
|
||||
$instance->save();
|
||||
}
|
||||
}
|
||||
}
|
43
app/Jobs/InstancePipeline/InstanceCrawlPipeline.php
Normal file
43
app/Jobs/InstancePipeline/InstanceCrawlPipeline.php
Normal file
|
@ -0,0 +1,43 @@
|
|||
<?php
|
||||
|
||||
namespace App\Jobs\InstancePipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use App\Instance;
|
||||
use App\Profile;
|
||||
use App\Services\NodeinfoService;
|
||||
|
||||
class InstanceCrawlPipeline implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
//
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
Instance::whereNull('last_crawled_at')->whereNull('software')->chunk(50, function($instances) use($headers) {
|
||||
foreach($instances as $instance) {
|
||||
FetchNodeinfoPipeline::dispatch($instance)->onQueue('low');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
76
app/Services/NodeinfoService.php
Normal file
76
app/Services/NodeinfoService.php
Normal file
|
@ -0,0 +1,76 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use Illuminate\Support\Str;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Http\Client\RequestException;
|
||||
use Illuminate\Http\Client\ConnectionException;
|
||||
|
||||
class NodeinfoService
|
||||
{
|
||||
public static function get($domain)
|
||||
{
|
||||
$version = config('pixelfed.version');
|
||||
$appUrl = config('app.url');
|
||||
$headers = [
|
||||
'Accept' => 'application/json',
|
||||
'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})",
|
||||
];
|
||||
|
||||
$url = 'https://' . $domain;
|
||||
$wk = $url . '/.well-known/nodeinfo';
|
||||
|
||||
try {
|
||||
$res = Http::withHeaders($headers)
|
||||
->timeout(5)
|
||||
->get($wk);
|
||||
} catch (RequestException $e) {
|
||||
return false;
|
||||
} catch (ConnectionException $e) {
|
||||
return false;
|
||||
} catch (\Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!$res) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$json = $res->json();
|
||||
|
||||
if( !isset($json['links'])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if(is_array($json['links'])) {
|
||||
if(isset($json['links']['href'])) {
|
||||
$href = $json['links']['href'];
|
||||
} else {
|
||||
$href = $json['links'][0]['href'];
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
$domain = parse_url($url, PHP_URL_HOST);
|
||||
$hrefDomain = parse_url($href, PHP_URL_HOST);
|
||||
|
||||
if($domain !== $hrefDomain) {
|
||||
return 60;
|
||||
}
|
||||
|
||||
try {
|
||||
$res = Http::withHeaders($headers)
|
||||
->timeout(5)
|
||||
->get($href);
|
||||
} catch (RequestException $e) {
|
||||
return false;
|
||||
} catch (ConnectionException $e) {
|
||||
return false;
|
||||
} catch (\Exception $e) {
|
||||
return false;
|
||||
}
|
||||
return $res->json();
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue