Add InstancePipeline and NodeinfoService

app/Console/Commands/StoryGC.php

@@ -7,6 +7,9 @@ use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Storage;
 use App\Story;
 use App\StoryView;
+use App\Jobs\StoryPipeline\StoryExpire;
+use App\Jobs\StoryPipeline\StoryRotateMedia;
+use App\Services\StoryService;
 
 class StoryGC extends Command
 {
@@ -41,89 +44,41 @@ class StoryGC extends Command
 	*/
 	public function handle()
 	{
-		$this->directoryScan();
+		$this->archiveExpiredStories();
-		$this->deleteViews();
+		$this->rotateMedia();
-		$this->deleteStories();
 	}
 
-	protected function directoryScan()
+	protected function archiveExpiredStories()
 	{
-		$day = now()->day;
+		$stories = Story::whereActive(true)
+		->where('expires_at', '<', now())
+		->get();
 
-		if($day !== 3) {
+		foreach($stories as $story) {
+			StoryExpire::dispatch($story)->onQueue('story');
+		}
+	}
+
+	protected function rotateMedia()
+	{
+		$queue = StoryService::rotateQueue();
+
+		if(!$queue || count($queue) == 0) {
 			return;
 		}
 
-		$monthHash = substr(hash('sha1', date('Y').date('m')), 0, 12);
+		collect($queue)
-
+			->each(function($id) {
-		$t1 = Storage::directories('public/_esm.t1');
+				$story = StoryService::getById($id);
-		$t2 = Storage::directories('public/_esm.t2');
+				if(!$story) {
-
+					StoryService::removeRotateQueue($id);
-		$dirs = array_merge($t1, $t2);
+					return;
-
-		foreach($dirs as $dir) {
-			$hash = last(explode('/', $dir));
-			if($hash != $monthHash) {
-				$this->info('Found directory to delete: ' . $dir);
-				$this->deleteDirectory($dir);
 				}
+				if($story->created_at->gt(now()->subMinutes(20))) {
+					return;
 				}
-
+				StoryRotateMedia::dispatch($story)->onQueue('story');
-		$mh = hash('sha256', date('Y').'-.-'.date('m'));
+				StoryService::removeRotateQueue($id);
-		$monthHash = date('Y').date('m').substr($mh, 0, 6).substr($mh, 58, 6);
-		$dirs = Storage::directories('public/_esm.t3');
-
-		foreach($dirs as $dir) {
-			$hash = last(explode('/', $dir));
-			if($hash != $monthHash) {
-				$this->info('Found directory to delete: ' . $dir);
-				$this->deleteDirectory($dir);
-			}
-		}
-	}
-
-	protected function deleteDirectory($path)
-	{
-		Storage::deleteDirectory($path);
-	}
-
-	protected function deleteViews()
-	{
-		StoryView::where('created_at', '<', now()->subMinutes(1441))->delete();
-	}
-
-	protected function deleteStories()
-	{
-		$stories = Story::where('created_at', '>', now()->subMinutes(30))
-		->whereNull('active')
-		->get();
-
-		foreach($stories as $story) {
-			if(Storage::exists($story->path) == true) {
-				Storage::delete($story->path);
-			}
-			DB::transaction(function() use($story) {
-				StoryView::whereStoryId($story->id)->delete();
-				$story->delete();
 			});
 	}
-
-		$stories = Story::where('created_at', '<', now()
-			->subMinutes(1441))
-		->get();
-
-		if($stories->count() == 0) {
-			exit;
-		}
-
-		foreach($stories as $story) {
-			if(Storage::exists($story->path) == true) {
-				Storage::delete($story->path);
-			}
-			DB::transaction(function() use($story) {
-				StoryView::whereStoryId($story->id)->delete();
-				$story->delete();
-			});
-		}
-	}
 }

app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace App\Jobs\InstancePipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Http;
+use App\Instance;
+use App\Profile;
+use App\Services\NodeinfoService;
+
+class FetchNodeinfoPipeline implements ShouldQueue
+{
+	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+	protected $instance;
+
+	/**
+	 * Create a new job instance.
+	 *
+	 * @return void
+	 */
+	public function __construct(Instance $instance)
+	{
+		$this->instance = $instance;
+	}
+
+	/**
+	 * Execute the job.
+	 *
+	 * @return void
+	 */
+	public function handle()
+	{
+		$instance = $this->instance;
+
+		$ni = NodeinfoService::get($instance->domain);
+		if($ni) {
+			if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) {
+				$software = $ni['software']['name'];
+				$instance->software = strtolower(strip_tags($software));
+				$instance->last_crawled_at = now();
+				$instance->user_count = Profile::whereDomain($instance->domain)->count();
+				$instance->save();
+			}
+		} else {
+			$instance->user_count = Profile::whereDomain($instance->domain)->count();
+			$instance->last_crawled_at = now();
+			$instance->save();
+		}
+	}
+}

app/Jobs/InstancePipeline/InstanceCrawlPipeline.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Jobs\InstancePipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Http;
+use App\Instance;
+use App\Profile;
+use App\Services\NodeinfoService;
+
+class InstanceCrawlPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        Instance::whereNull('last_crawled_at')->whereNull('software')->chunk(50, function($instances) use($headers) {
+			foreach($instances as $instance) {
+				FetchNodeinfoPipeline::dispatch($instance)->onQueue('low');
+			}
+		});
+    }
+}

app/Services/NodeinfoService.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Str;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Http\Client\ConnectionException;
+
+class NodeinfoService
+{
+    public static function get($domain)
+    {
+    	$version = config('pixelfed.version');
+		$appUrl = config('app.url');
+		$headers = [
+			'Accept'     => 'application/json',
+			'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})",
+		];
+
+        $url = 'https://' . $domain;
+        $wk = $url . '/.well-known/nodeinfo';
+
+        try {
+            $res = Http::withHeaders($headers)
+            ->timeout(5)
+            ->get($wk);
+        } catch (RequestException $e) {
+            return false;
+        } catch (ConnectionException $e) {
+            return false;
+        } catch (\Exception $e) {
+            return false;
+        }
+
+        if(!$res) {
+            return false;
+        }
+
+        $json = $res->json();
+
+        if( !isset($json['links'])) {
+            return false;
+        }
+
+        if(is_array($json['links'])) {
+            if(isset($json['links']['href'])) {
+                $href = $json['links']['href'];
+            } else {
+                $href = $json['links'][0]['href'];
+            }
+        } else {
+            return false;
+        }
+
+        $domain = parse_url($url, PHP_URL_HOST);
+        $hrefDomain = parse_url($href, PHP_URL_HOST);
+
+        if($domain !== $hrefDomain) {
+            return 60;
+        }
+
+        try {
+            $res = Http::withHeaders($headers)
+            ->timeout(5)
+            ->get($href);
+        } catch (RequestException $e) {
+            return false;
+        } catch (ConnectionException $e) {
+            return false;
+        } catch (\Exception $e) {
+            return false;
+        }
+        return $res->json();
+    }
+}