mirror of
https://github.com/pixelfed/pixelfed.git
synced 2024-11-09 16:24:51 +00:00
Add InstancePipeline and NodeinfoService
This commit is contained in:
parent
7641b73158
commit
da6943daed
4 changed files with 206 additions and 76 deletions
|
@ -7,6 +7,9 @@ use Illuminate\Support\Facades\DB;
|
||||||
use Illuminate\Support\Facades\Storage;
|
use Illuminate\Support\Facades\Storage;
|
||||||
use App\Story;
|
use App\Story;
|
||||||
use App\StoryView;
|
use App\StoryView;
|
||||||
|
use App\Jobs\StoryPipeline\StoryExpire;
|
||||||
|
use App\Jobs\StoryPipeline\StoryRotateMedia;
|
||||||
|
use App\Services\StoryService;
|
||||||
|
|
||||||
class StoryGC extends Command
|
class StoryGC extends Command
|
||||||
{
|
{
|
||||||
|
@ -41,89 +44,41 @@ class StoryGC extends Command
|
||||||
*/
|
*/
|
||||||
public function handle()
|
public function handle()
|
||||||
{
|
{
|
||||||
$this->directoryScan();
|
$this->archiveExpiredStories();
|
||||||
$this->deleteViews();
|
$this->rotateMedia();
|
||||||
$this->deleteStories();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function directoryScan()
|
protected function archiveExpiredStories()
|
||||||
{
|
{
|
||||||
$day = now()->day;
|
$stories = Story::whereActive(true)
|
||||||
|
->where('expires_at', '<', now())
|
||||||
|
->get();
|
||||||
|
|
||||||
if($day !== 3) {
|
foreach($stories as $story) {
|
||||||
|
StoryExpire::dispatch($story)->onQueue('story');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function rotateMedia()
|
||||||
|
{
|
||||||
|
$queue = StoryService::rotateQueue();
|
||||||
|
|
||||||
|
if(!$queue || count($queue) == 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
$monthHash = substr(hash('sha1', date('Y').date('m')), 0, 12);
|
collect($queue)
|
||||||
|
->each(function($id) {
|
||||||
$t1 = Storage::directories('public/_esm.t1');
|
$story = StoryService::getById($id);
|
||||||
$t2 = Storage::directories('public/_esm.t2');
|
if(!$story) {
|
||||||
|
StoryService::removeRotateQueue($id);
|
||||||
$dirs = array_merge($t1, $t2);
|
return;
|
||||||
|
}
|
||||||
foreach($dirs as $dir) {
|
if($story->created_at->gt(now()->subMinutes(20))) {
|
||||||
$hash = last(explode('/', $dir));
|
return;
|
||||||
if($hash != $monthHash) {
|
}
|
||||||
$this->info('Found directory to delete: ' . $dir);
|
StoryRotateMedia::dispatch($story)->onQueue('story');
|
||||||
$this->deleteDirectory($dir);
|
StoryService::removeRotateQueue($id);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$mh = hash('sha256', date('Y').'-.-'.date('m'));
|
|
||||||
$monthHash = date('Y').date('m').substr($mh, 0, 6).substr($mh, 58, 6);
|
|
||||||
$dirs = Storage::directories('public/_esm.t3');
|
|
||||||
|
|
||||||
foreach($dirs as $dir) {
|
|
||||||
$hash = last(explode('/', $dir));
|
|
||||||
if($hash != $monthHash) {
|
|
||||||
$this->info('Found directory to delete: ' . $dir);
|
|
||||||
$this->deleteDirectory($dir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function deleteDirectory($path)
|
|
||||||
{
|
|
||||||
Storage::deleteDirectory($path);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function deleteViews()
|
|
||||||
{
|
|
||||||
StoryView::where('created_at', '<', now()->subMinutes(1441))->delete();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function deleteStories()
|
|
||||||
{
|
|
||||||
$stories = Story::where('created_at', '>', now()->subMinutes(30))
|
|
||||||
->whereNull('active')
|
|
||||||
->get();
|
|
||||||
|
|
||||||
foreach($stories as $story) {
|
|
||||||
if(Storage::exists($story->path) == true) {
|
|
||||||
Storage::delete($story->path);
|
|
||||||
}
|
|
||||||
DB::transaction(function() use($story) {
|
|
||||||
StoryView::whereStoryId($story->id)->delete();
|
|
||||||
$story->delete();
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
$stories = Story::where('created_at', '<', now()
|
|
||||||
->subMinutes(1441))
|
|
||||||
->get();
|
|
||||||
|
|
||||||
if($stories->count() == 0) {
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach($stories as $story) {
|
|
||||||
if(Storage::exists($story->path) == true) {
|
|
||||||
Storage::delete($story->path);
|
|
||||||
}
|
|
||||||
DB::transaction(function() use($story) {
|
|
||||||
StoryView::whereStoryId($story->id)->delete();
|
|
||||||
$story->delete();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
56
app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
Normal file
56
app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php
Normal file
|
@ -0,0 +1,56 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Jobs\InstancePipeline;
|
||||||
|
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||||||
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
use Illuminate\Support\Facades\Http;
|
||||||
|
use App\Instance;
|
||||||
|
use App\Profile;
|
||||||
|
use App\Services\NodeinfoService;
|
||||||
|
|
||||||
|
class FetchNodeinfoPipeline implements ShouldQueue
|
||||||
|
{
|
||||||
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
|
protected $instance;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new job instance.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function __construct(Instance $instance)
|
||||||
|
{
|
||||||
|
$this->instance = $instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the job.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function handle()
|
||||||
|
{
|
||||||
|
$instance = $this->instance;
|
||||||
|
|
||||||
|
$ni = NodeinfoService::get($instance->domain);
|
||||||
|
if($ni) {
|
||||||
|
if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) {
|
||||||
|
$software = $ni['software']['name'];
|
||||||
|
$instance->software = strtolower(strip_tags($software));
|
||||||
|
$instance->last_crawled_at = now();
|
||||||
|
$instance->user_count = Profile::whereDomain($instance->domain)->count();
|
||||||
|
$instance->save();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$instance->user_count = Profile::whereDomain($instance->domain)->count();
|
||||||
|
$instance->last_crawled_at = now();
|
||||||
|
$instance->save();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
43
app/Jobs/InstancePipeline/InstanceCrawlPipeline.php
Normal file
43
app/Jobs/InstancePipeline/InstanceCrawlPipeline.php
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Jobs\InstancePipeline;
|
||||||
|
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||||||
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
use Illuminate\Support\Facades\Http;
|
||||||
|
use App\Instance;
|
||||||
|
use App\Profile;
|
||||||
|
use App\Services\NodeinfoService;
|
||||||
|
|
||||||
|
class InstanceCrawlPipeline implements ShouldQueue
|
||||||
|
{
|
||||||
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new job instance.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function __construct()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the job.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function handle()
|
||||||
|
{
|
||||||
|
Instance::whereNull('last_crawled_at')->whereNull('software')->chunk(50, function($instances) use($headers) {
|
||||||
|
foreach($instances as $instance) {
|
||||||
|
FetchNodeinfoPipeline::dispatch($instance)->onQueue('low');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
76
app/Services/NodeinfoService.php
Normal file
76
app/Services/NodeinfoService.php
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Services;
|
||||||
|
|
||||||
|
use Illuminate\Support\Str;
|
||||||
|
use Illuminate\Support\Facades\Http;
|
||||||
|
use Illuminate\Http\Client\RequestException;
|
||||||
|
use Illuminate\Http\Client\ConnectionException;
|
||||||
|
|
||||||
|
class NodeinfoService
|
||||||
|
{
|
||||||
|
public static function get($domain)
|
||||||
|
{
|
||||||
|
$version = config('pixelfed.version');
|
||||||
|
$appUrl = config('app.url');
|
||||||
|
$headers = [
|
||||||
|
'Accept' => 'application/json',
|
||||||
|
'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})",
|
||||||
|
];
|
||||||
|
|
||||||
|
$url = 'https://' . $domain;
|
||||||
|
$wk = $url . '/.well-known/nodeinfo';
|
||||||
|
|
||||||
|
try {
|
||||||
|
$res = Http::withHeaders($headers)
|
||||||
|
->timeout(5)
|
||||||
|
->get($wk);
|
||||||
|
} catch (RequestException $e) {
|
||||||
|
return false;
|
||||||
|
} catch (ConnectionException $e) {
|
||||||
|
return false;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!$res) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$json = $res->json();
|
||||||
|
|
||||||
|
if( !isset($json['links'])) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(is_array($json['links'])) {
|
||||||
|
if(isset($json['links']['href'])) {
|
||||||
|
$href = $json['links']['href'];
|
||||||
|
} else {
|
||||||
|
$href = $json['links'][0]['href'];
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$domain = parse_url($url, PHP_URL_HOST);
|
||||||
|
$hrefDomain = parse_url($href, PHP_URL_HOST);
|
||||||
|
|
||||||
|
if($domain !== $hrefDomain) {
|
||||||
|
return 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$res = Http::withHeaders($headers)
|
||||||
|
->timeout(5)
|
||||||
|
->get($href);
|
||||||
|
} catch (RequestException $e) {
|
||||||
|
return false;
|
||||||
|
} catch (ConnectionException $e) {
|
||||||
|
return false;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return $res->json();
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue