Improve emoji import

This commit is contained in:
Daniel Supernault 2022-01-19 00:46:30 -07:00
parent 01798daf56
commit dc17c9fc27
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
5 changed files with 81 additions and 17 deletions

View file

@ -0,0 +1,56 @@
<?php
namespace App\Jobs\StatusPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\CustomEmojiService;
use App\Services\StatusService;
class StatusTagsPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $activity;
protected $status;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($activity, $status)
{
$this->activity = $activity;
$this->status = $status;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$res = $this->activity;
collect($res['tag'])
->filter(function($tag) {
// todo: finish hashtag + mention import
// return in_array($tag['type'], ['Emoji', 'Hashtag', 'Mention']);
return $tag && $tag['type'] == 'Emoji';
})
->values()
->map(function($tag) {
CustomEmojiService::import($tag['id']);
});
// sleep(15);
StatusService::del($this->status->id);
}
}

View file

@ -16,6 +16,10 @@ class CustomEmoji extends Model
public static function scan($text) public static function scan($text)
{ {
if(config('federation.custom_emoji.enabled') == false) {
return [];
}
return Str::of($text) return Str::of($text)
->matchAll(self::SCAN_RE) ->matchAll(self::SCAN_RE)
->map(function($match) { ->map(function($match) {

View file

@ -2,26 +2,35 @@
namespace App\Services; namespace App\Services;
use App\Models\CustomEmoji;
use App\Util\ActivityPub\Helpers; use App\Util\ActivityPub\Helpers;
use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Http;
use App\Models\CustomEmoji; use Illuminate\Support\Facades\Cache;
class CustomEmojiService class CustomEmojiService
{ {
public static function get($shortcode) public static function get($shortcode)
{ {
if(config('federation.custom_emoji.enabled') == false) {
return;
}
return CustomEmoji::whereShortcode($shortcode)->first(); return CustomEmoji::whereShortcode($shortcode)->first();
} }
public static function import($url) public static function import($url)
{ {
if(config('federation.custom_emoji.enabled') == false) {
return;
}
if(Helpers::validateUrl($url) == false) { if(Helpers::validateUrl($url) == false) {
return; return;
} }
$emoji = CustomEmoji::whereUri($url)->first(); $emoji = CustomEmoji::whereUri($url)->first();
if($emoji) { if($emoji) {
return $emoji; return;
} }
$res = Http::acceptJson()->get($url); $res = Http::acceptJson()->get($url);
@ -47,6 +56,7 @@ class CustomEmojiService
if(!self::headCheck($json['icon']['url'])) { if(!self::headCheck($json['icon']['url'])) {
return; return;
} }
$emoji = new CustomEmoji; $emoji = new CustomEmoji;
$emoji->shortcode = $json['name']; $emoji->shortcode = $json['name'];
$emoji->uri = $json['id']; $emoji->uri = $json['id'];
@ -60,7 +70,9 @@ class CustomEmojiService
$emoji->media_path = 'emoji/' . $emoji->id . $ext; $emoji->media_path = 'emoji/' . $emoji->id . $ext;
$emoji->save(); $emoji->save();
return $emoji; $name = str_replace(':', '', $json['name']);
Cache::forget('pf:custom_emoji:' . $name);
return;
} else { } else {
return; return;
} }

View file

@ -23,6 +23,7 @@ use App\Jobs\RemoteFollowPipeline\RemoteFollowImportRecent;
use App\Jobs\ImageOptimizePipeline\{ImageOptimize,ImageThumbnail}; use App\Jobs\ImageOptimizePipeline\{ImageOptimize,ImageThumbnail};
use App\Jobs\StatusPipeline\NewStatusPipeline; use App\Jobs\StatusPipeline\NewStatusPipeline;
use App\Jobs\StatusPipeline\StatusReplyPipeline; use App\Jobs\StatusPipeline\StatusReplyPipeline;
use App\Jobs\StatusPipeline\StatusTagsPipeline;
use App\Util\ActivityPub\HttpSignature; use App\Util\ActivityPub\HttpSignature;
use Illuminate\Support\Str; use Illuminate\Support\Str;
use App\Services\ActivityPubFetchService; use App\Services\ActivityPubFetchService;
@ -381,19 +382,6 @@ class Helpers {
$scope, $scope,
$id $id
) { ) {
if(isset($res['tag']) && is_array($res['tag']) && !empty($res['tag'])) {
collect($res['tag'])
->filter(function($tag) {
// todo: finish hashtag + mention import
// return in_array($tag['type'], ['Emoji', 'Hashtag', 'Mention']);
return in_array($tag['type'], ['Emoji']);
})
->each(function($tag) {
if(isset($tag['id'])) {
CustomEmojiService::import($tag['id']);
}
});
}
if($res['type'] === 'Question') { if($res['type'] === 'Question') {
$status = self::storePoll( $status = self::storePoll(
@ -430,6 +418,10 @@ class Helpers {
} else { } else {
StatusReplyPipeline::dispatch($status); StatusReplyPipeline::dispatch($status);
} }
if(isset($res['tag']) && is_array($res['tag']) && !empty($res['tag'])) {
StatusTagsPipeline::dispatch($res, $status);
}
return $status; return $status;
}); });
}); });

View file

@ -15,7 +15,7 @@ class CreateCustomEmojiTable extends Migration
{ {
Schema::create('custom_emoji', function (Blueprint $table) { Schema::create('custom_emoji', function (Blueprint $table) {
$table->id(); $table->id();
$table->string('shortcode')->unique()->index(); $table->string('shortcode')->index();
$table->string('media_path')->nullable(); $table->string('media_path')->nullable();
$table->string('domain')->nullable()->index(); $table->string('domain')->nullable()->index();
$table->boolean('disabled')->default(false)->index(); $table->boolean('disabled')->default(false)->index();