From 132a58de5419e6af0ed4ebe9b8d93bba96a5448b Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 17 May 2023 04:00:03 -0600 Subject: [PATCH] Add Autospam Advanced Detection --- .../Admin/AdminAutospamController.php | 255 ++++++++++++++++++ app/Http/Controllers/AdminController.php | 2 + .../AutospamPretrainNonSpamPipeline.php | 58 ++++ .../AutospamPretrainPipeline.php | 63 +++++ .../AutospamUpdateCachedDataPipeline.php | 79 ++++++ app/Services/AutospamService.php | 78 ++++++ app/Services/ConfigCacheService.php | 4 +- app/Util/Lexer/Classifier.php | 178 ++++++++++++ public/js/admin.js | Bin 164840 -> 213614 bytes public/mix-manifest.json | Bin 5977 -> 5977 bytes resources/views/admin/autospam/home.blade.php | 12 + .../views/admin/settings/sidebar.blade.php | 105 +++++--- routes/web.php | 13 + storage/app/.gitignore | 1 + storage/app/nlp/.gitignore | 2 + 15 files changed, 819 insertions(+), 31 deletions(-) create mode 100644 app/Http/Controllers/Admin/AdminAutospamController.php create mode 100644 app/Jobs/AutospamPipeline/AutospamPretrainNonSpamPipeline.php create mode 100644 app/Jobs/AutospamPipeline/AutospamPretrainPipeline.php create mode 100644 app/Jobs/AutospamPipeline/AutospamUpdateCachedDataPipeline.php create mode 100644 app/Services/AutospamService.php create mode 100644 app/Util/Lexer/Classifier.php create mode 100644 resources/views/admin/autospam/home.blade.php create mode 100644 storage/app/nlp/.gitignore diff --git a/app/Http/Controllers/Admin/AdminAutospamController.php b/app/Http/Controllers/Admin/AdminAutospamController.php new file mode 100644 index 000000000..8adc45312 --- /dev/null +++ b/app/Http/Controllers/Admin/AdminAutospamController.php @@ -0,0 +1,255 @@ +whereNull('appeal_handled_at')->count(); + }); + + $closed = Cache::remember('admin-dash:reports:spam-count-closed', 3600, function() { + return AccountInterstitial::whereType('post.autospam')->whereNotNull('appeal_handled_at')->count(); + }); + + $thisWeek = Cache::remember('admin-dash:reports:spam-count-stats-this-week ', 86400, function() { + $sr = config('database.default') == 'pgsql' ? "to_char(created_at, 'MM-YYYY')" : "DATE_FORMAT(created_at, '%m-%Y')"; + $gb = config('database.default') == 'pgsql' ? [DB::raw($sr)] : DB::raw($sr); + $s = AccountInterstitial::select( + DB::raw('count(id) as count'), + DB::raw($sr . " as month_year") + ) + ->where('created_at', '>=', now()->subWeeks(52)) + ->groupBy($gb) + ->get() + ->map(function($s) { + $dt = now()->parse('01-' . $s->month_year); + return [ + 'id' => $dt->format('Ym'), + 'x' => $dt->format('M Y'), + 'y' => $s->count + ]; + }) + ->sortBy('id') + ->values() + ->toArray(); + return $s; + }); + + $files = [ + 'spam' => [ + 'exists' => Storage::exists(AutospamService::MODEL_SPAM_PATH), + 'size' => 0 + ], + 'ham' => [ + 'exists' => Storage::exists(AutospamService::MODEL_HAM_PATH), + 'size' => 0 + ], + 'combined' => [ + 'exists' => Storage::exists(AutospamService::MODEL_FILE_PATH), + 'size' => 0 + ] + ]; + + if($files['spam']['exists']) { + $files['spam']['size'] = Storage::size(AutospamService::MODEL_SPAM_PATH); + } + + if($files['ham']['exists']) { + $files['ham']['size'] = Storage::size(AutospamService::MODEL_HAM_PATH); + } + + if($files['combined']['exists']) { + $files['combined']['size'] = Storage::size(AutospamService::MODEL_FILE_PATH); + } + + return [ + 'autospam_enabled' => (bool) config_cache('pixelfed.bouncer.enabled') ?? false, + 'nlp_enabled' => (bool) AutospamService::active(), + 'files' => $files, + 'open' => $open, + 'closed' => $closed, + 'graph' => collect($thisWeek)->map(fn($s) => $s['y'])->values(), + 'graphLabels' => collect($thisWeek)->map(fn($s) => $s['x'])->values() + ]; + } + + public function getAutospamReportsClosedApi(Request $request) + { + $appeals = AdminSpamReport::collection( + AccountInterstitial::orderBy('id', 'desc') + ->whereType('post.autospam') + ->whereIsSpam(true) + ->whereNotNull('appeal_handled_at') + ->cursorPaginate(6) + ->withQueryString() + ); + + return $appeals; + } + + public function postAutospamTrainSpamApi(Request $request) + { + $aiCount = AccountInterstitial::whereItemType('App\Status') + ->whereIsSpam(true) + ->count(); + abort_if($aiCount < 100, 422, 'You don\'t have enough data to pre-train against.'); + + $existing = Cache::get('pf:admin:autospam:pretrain:recent'); + abort_if($existing, 422, 'You\'ve already run this recently, please wait 30 minutes before pre-training again'); + AutospamPretrainPipeline::dispatch(); + Cache::put('pf:admin:autospam:pretrain:recent', 1, 1440); + + return [ + 'msg' => 'Success!' + ]; + } + + public function postAutospamTrainNonSpamSearchApi(Request $request) + { + $this->validate($request, [ + 'q' => 'required|string|min:1' + ]); + + $q = $request->input('q'); + + $res = Profile::whereNull(['status', 'domain']) + ->where('username', 'like', '%' . $q . '%') + ->orderByDesc('followers_count') + ->take(10) + ->get() + ->map(function($p) { + $acct = AccountService::get($p->id, true); + return [ + 'id' => (string) $p->id, + 'avatar' => $acct['avatar'], + 'username' => $p->username + ]; + }) + ->values(); + return $res; + } + + public function postAutospamTrainNonSpamSubmitApi(Request $request) + { + $this->validate($request, [ + 'accounts' => 'required|array|min:1|max:10' + ]); + + $accts = $request->input('accounts'); + + $accounts = Profile::whereNull(['domain', 'status'])->find(collect($accts)->map(function($a) { return $a['id'];})); + + abort_if(!$accounts || !$accounts->count(), 422, 'One or more of the selected accounts are not valid'); + + AutospamPretrainNonSpamPipeline::dispatch($accounts); + return $accounts; + } + + public function getAutospamCustomTokensApi(Request $request) + { + return AutospamCustomTokens::latest()->cursorPaginate(6); + } + + public function saveNewAutospamCustomTokensApi(Request $request) + { + $this->validate($request, [ + 'token' => 'required|unique:autospam_custom_tokens,token', + ]); + + $ct = new AutospamCustomTokens; + $ct->token = $request->input('token'); + $ct->weight = $request->input('weight'); + $ct->category = $request->input('category') === 'spam' ? 'spam' : 'ham'; + $ct->note = $request->input('note'); + $ct->active = $request->input('active'); + $ct->save(); + + AutospamUpdateCachedDataPipeline::dispatch(); + return $ct; + } + + public function updateAutospamCustomTokensApi(Request $request) + { + $this->validate($request, [ + 'id' => 'required', + 'token' => 'required', + 'category' => 'required|in:spam,ham', + 'active' => 'required|boolean' + ]); + + $ct = AutospamCustomTokens::findOrFail($request->input('id')); + $ct->weight = $request->input('weight'); + $ct->category = $request->input('category'); + $ct->note = $request->input('note'); + $ct->active = $request->input('active'); + $ct->save(); + + AutospamUpdateCachedDataPipeline::dispatch(); + + return $ct; + } + + public function exportAutospamCustomTokensApi(Request $request) + { + abort_if(!Storage::exists(AutospamService::MODEL_SPAM_PATH), 422, 'Autospam Dataset does not exist, please train spam before attempting to export'); + return Storage::download(AutospamService::MODEL_SPAM_PATH); + } + + public function enableAutospamApi(Request $request) + { + ConfigCacheService::put('autospam.nlp.enabled', true); + Cache::forget(AutospamService::CHCKD_CACHE_KEY); + return ['msg' => 'Success']; + } + + public function disableAutospamApi(Request $request) + { + ConfigCacheService::put('autospam.nlp.enabled', false); + Cache::forget(AutospamService::CHCKD_CACHE_KEY); + return ['msg' => 'Success']; + } +} diff --git a/app/Http/Controllers/AdminController.php b/app/Http/Controllers/AdminController.php index caa9c19fc..e54908a41 100644 --- a/app/Http/Controllers/AdminController.php +++ b/app/Http/Controllers/AdminController.php @@ -21,6 +21,7 @@ use Carbon\Carbon; use Illuminate\Http\Request; use Illuminate\Support\Facades\Redis; use App\Http\Controllers\Admin\{ + AdminAutospamController, AdminDirectoryController, AdminDiscoverController, AdminHashtagsController, @@ -43,6 +44,7 @@ use App\Models\CustomEmoji; class AdminController extends Controller { use AdminReportController, + AdminAutospamController, AdminDirectoryController, AdminDiscoverController, AdminHashtagsController, diff --git a/app/Jobs/AutospamPipeline/AutospamPretrainNonSpamPipeline.php b/app/Jobs/AutospamPipeline/AutospamPretrainNonSpamPipeline.php new file mode 100644 index 000000000..348f8e0ea --- /dev/null +++ b/app/Jobs/AutospamPipeline/AutospamPretrainNonSpamPipeline.php @@ -0,0 +1,58 @@ +accounts = $accounts; + $this->classifier = new Classifier(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $classifier = $this->classifier; + $accounts = $this->accounts; + + foreach($accounts as $acct) { + Status::whereNotNull('caption') + ->whereScope('public') + ->whereProfileId($acct->id) + ->inRandomOrder() + ->take(400) + ->pluck('caption') + ->each(function($c) use ($classifier) { + $classifier->learn($c, 'ham'); + }); + } + + Storage::put(AutospamService::MODEL_HAM_PATH, $classifier->export()); + + AutospamUpdateCachedDataPipeline::dispatch()->delay(5); + } +} diff --git a/app/Jobs/AutospamPipeline/AutospamPretrainPipeline.php b/app/Jobs/AutospamPipeline/AutospamPretrainPipeline.php new file mode 100644 index 000000000..f1f637c37 --- /dev/null +++ b/app/Jobs/AutospamPipeline/AutospamPretrainPipeline.php @@ -0,0 +1,63 @@ +classifier = new Classifier(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $classifier = $this->classifier; + + $aiCount = AccountInterstitial::whereItemType('App\Status') + ->whereIsSpam(true) + ->count(); + + if($aiCount < 100) { + return; + } + + AccountInterstitial::whereItemType('App\Status') + ->whereIsSpam(true) + ->inRandomOrder() + ->take(config('autospam.nlp.spam_sample_limit')) + ->pluck('item_id') + ->each(function ($ai) use($classifier) { + $status = Status::whereNotNull('caption')->find($ai); + if(!$status) { + return; + } + $classifier->learn($status->caption, 'spam'); + }); + + Storage::put(AutospamService::MODEL_SPAM_PATH, $classifier->export()); + + AutospamUpdateCachedDataPipeline::dispatch()->delay(5); + } +} diff --git a/app/Jobs/AutospamPipeline/AutospamUpdateCachedDataPipeline.php b/app/Jobs/AutospamPipeline/AutospamUpdateCachedDataPipeline.php new file mode 100644 index 000000000..c223e74f7 --- /dev/null +++ b/app/Jobs/AutospamPipeline/AutospamUpdateCachedDataPipeline.php @@ -0,0 +1,79 @@ +get(); + foreach($newSpam as $ns) { + $key = strtolower($ns->token); + if(isset($spam['words']['spam'][$key])) { + $spam['words']['spam'][$key] = $spam['words']['spam'][$key] + $ns->weight; + } else { + $spam['words']['spam'][$key] = $ns->weight; + } + } + $newSpamCount = count($spam['words']['spam']); + $spam['documents']['spam'] = $newSpamCount; + arsort($spam['words']['spam']); + Storage::put(AutospamService::MODEL_SPAM_PATH, json_encode($spam, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT)); + + $ham = json_decode(Storage::get(AutospamService::MODEL_HAM_PATH), true); + $newHam = AutospamCustomTokens::whereCategory('ham')->get(); + foreach($newHam as $ns) { + $key = strtolower($ns->token); + if(isset($spam['words']['ham'][$key])) { + $ham['words']['ham'][$key] = $ham['words']['ham'][$key] + $ns->weight; + } else { + $ham['words']['ham'][$key] = $ns->weight; + } + } + + $newHamCount = count($ham['words']['ham']); + $ham['documents']['ham'] = $newHamCount; + arsort($ham['words']['ham']); + + Storage::put(AutospamService::MODEL_HAM_PATH, json_encode($ham, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT)); + + $combined = [ + 'documents' => [ + 'spam' => $newSpamCount, + 'ham' => $newHamCount, + ], + 'words' => [ + 'spam' => $spam['words']['spam'], + 'ham' => $ham['words']['ham'] + ] + ]; + + Storage::put(AutospamService::MODEL_FILE_PATH, json_encode($combined, JSON_PRETTY_PRINT,JSON_UNESCAPED_SLASHES)); + Cache::forget(AutospamService::MODEL_CACHE_KEY); + Cache::forget(AutospamService::CHCKD_CACHE_KEY); + } +} diff --git a/app/Services/AutospamService.php b/app/Services/AutospamService.php new file mode 100644 index 000000000..6986e81e4 --- /dev/null +++ b/app/Services/AutospamService.php @@ -0,0 +1,78 @@ +import($model['documents'], $model['words']); + return $classifier->most($text) === 'spam'; + } + + public static function eligible() + { + return Cache::remember(self::CHCKD_CACHE_KEY, 86400, function() { + if(!config_cache('pixelfed.bouncer.enabled') || !config('autospam.enabled')) { + return false; + } + + if(!Storage::exists(self::MODEL_SPAM_PATH)) { + return false; + } + + if(!Storage::exists(self::MODEL_HAM_PATH)) { + return false; + } + + if(!Storage::exists(self::MODEL_FILE_PATH)) { + return false; + } else { + if(Storage::size(self::MODEL_FILE_PATH) < 1000) { + return false; + } + } + + return true; + }); + } + + public static function active() + { + return config_cache('autospam.nlp.enabled') && self::eligible(); + } + + public static function getCachedModel() + { + if(!self::active()) { + return null; + } + + return Cache::remember(self::MODEL_CACHE_KEY, 86400, function() { + $res = Storage::get(self::MODEL_FILE_PATH); + if(!$res || empty($res)) { + return null; + } + + return json_decode($res, true); + }); + } +} diff --git a/app/Services/ConfigCacheService.php b/app/Services/ConfigCacheService.php index 7ecb318e0..9da5c8adc 100644 --- a/app/Services/ConfigCacheService.php +++ b/app/Services/ConfigCacheService.php @@ -69,7 +69,9 @@ class ConfigCacheService 'instance.landing.show_directory', 'instance.landing.show_explore', 'instance.admin.pid', - 'instance.banner.blurhash' + 'instance.banner.blurhash', + + 'autospam.nlp.enabled', // 'system.user_mode' ]; diff --git a/app/Util/Lexer/Classifier.php b/app/Util/Lexer/Classifier.php new file mode 100644 index 000000000..61f7b694c --- /dev/null +++ b/app/Util/Lexer/Classifier.php @@ -0,0 +1,178 @@ + + */ + private $tokenizer; + + /** + * @var array> + */ + private array $words = []; + + /** + * @var array + */ + private array $documents = []; + + private bool $uneven = false; + + /** + * @param callable(string): array $tokenizer + */ + public function setTokenizer(callable $tokenizer): void + { + $this->tokenizer = $tokenizer; + } + + /** + * @return Collection + */ + public function tokenize(string $string): Collection + { + if ($this->tokenizer) { + /** @var array */ + $tokens = call_user_func($this->tokenizer, $string); + + return collect($tokens); + } + + return Str::of($string) + ->lower() + ->matchAll('/[[:alpha:]]+/u'); + } + + /** + * @return $this + */ + public function learn(string $statement, string $type): self + { + foreach ($this->tokenize($statement) as $word) { + $this->incrementWord($type, $word); + } + + $this->incrementType($type); + + return $this; + } + + /** + * @return Collection + */ + public function guess(string $statement): Collection + { + $words = $this->tokenize($statement); + + return collect($this->documents) + ->map(function ($count, string $type) use ($words) { + $likelihood = $this->pTotal($type); + + foreach ($words as $word) { + $likelihood *= $this->p($word, $type); + } + + return (string) BigDecimal::of($likelihood); + }) + ->sortDesc(); + } + + public function most(string $statement): string + { + /** @var string */ + return $this->guess($statement)->keys()->first(); + } + + /** + * @return self + */ + public function uneven(bool $enabled = false): self + { + $this->uneven = $enabled; + + return $this; + } + + /** + * Increment the document count for the type + */ + private function incrementType(string $type): void + { + if (! isset($this->documents[$type])) { + $this->documents[$type] = 0; + } + + $this->documents[$type]++; + } + + /** + * Increment the word count for the given type + */ + private function incrementWord(string $type, string $word): void + { + $ignored = config('autospam.ignored_tokens'); + if(!$ignored) { + $ignored = ['the', 'a', 'of', 'and']; + } else { + $ignored = explode(',', $ignored); + } + if ($type == 'spam' && in_array($word, $ignored)) { + return; + } + if (! isset($this->words[$type][$word])) { + $this->words[$type][$word] = 0; + } + + $this->words[$type][$word]++; + } + + /** + * @return float|int + */ + private function p(string $word, string $type) + { + $count = $this->words[$type][$word] ?? 0; + + return ($count + 1) / (array_sum($this->words[$type]) + 1); + } + + /** + * @return float|int + */ + private function pTotal(string $type) + { + return $this->uneven + ? ($this->documents[$type] + 1) / (array_sum($this->documents) + 1) + : 1; + } + + public function export() + { + $words = $this->words; + $words = collect($words) + ->map(function($w) { + arsort($w); + return $w; + }) + ->all(); + return json_encode([ + '_ns' => 'https://pixelfed.org/ns/nlp', + '_v' => '1.0', + 'documents' => $this->documents, + 'words' => $words + ], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES); + } + + public function import($documents, $words) + { + $this->documents = $documents; + $this->words = $words; + } +} diff --git a/public/js/admin.js b/public/js/admin.js index b2efdf1dd798f26f4ad267323289e9939de19ac6..48d5949b877eac5b90d638580810e91be004a0f6 100644 GIT binary patch delta 19058 zcmeHv3vgWJeeZk+$uGSvSx?J8du+?Q_-L)w%kl~#3)z5!Ez2($V>{+(_ei_(?w<9Y zvyz3Z3OAiLm&_%=m*Gq1GGK1+O_+od9BwNvE(Dr8ArQ)=32A#fNul>|0t0o^fRF_Nz!kIq9jfI-ScFGeyQ6}M(L~FJ!F<%)00wW{FbT*<|ebL zh!r{S2hMG|mfw^fdBS(gkPtu;F64_7xPv00;>F<9RD{kF&h@_vicKtS4n;zuj z;zLRL;lODOIKBI`tI~mRD(t5_cQ?`U@C|6ZIs932jD9aXjx%i=948~cKRrmQ_Rr6I z`|hWozHS#8ee%k64-*{l z{!!jBszy}hCOxHRG|R{+qKQ5@EMuW(hW}~fhwR1skljBx#<)K$r&0SJFh&L{rzi^XZ75 zHZqnOK6$#Z;YQtx#4ykoDr_IkW=tGhENQRhtT1|mmO5Z)(St@t=d;Y1F>@5Bu)7Z$ zQ7y4dGoKYtO@)13^c#2fEr^+8Mv71NrlsufBaw)aP1(mabuAN#d6RL@tZ5m^W5!uM zW#1?7d;WFunn#zOd0{l8YnCpS@wU%f`bHy@49{77m+-U^kIG%@j2@qgSz(+26Q_*K zY*^v&SJaeY>Fyv61U?PO0P-9(`=W8Ha1R|VqgnH`dx!Acyy~>MO&@=6`jqUWe?8Mg zQxz4Bsui5%XdV@W;}6US)R5joKU1#IOAD}x=_~>^kTd8qkup;lilql`=Y_z_{<(Hi{1JT-5O)d7v zAw3w?EDdlr`3cup(o6xT?a(_{5+1Hhlg^32>PL~ znf~>yH!B{D9K`Olv8fDd=i`PMOdBSB^*^*!EH@r6p)&Z>oYWU zZxdbj=}J2E*-y}iKizo^$cWx~_jt5mVlWX;oi&3g{k-K|AfH&A-gHKv9uzmvyN63p zI7KM#3;xmdBl;YK1EeV3mfg$qkYcXV`yhQNLWi_krl+o~al4o&*O zeNlct`jD!YWlcRpUs5|7`5bYdCDRu2rgd~I9b;NdwZ`<7g`| zS8@?Eorp(t7&W27g?i#CXQiXriDcZ$t>i4U6ayhT($>1mgTA1seBf)YE-ZBrdT^%g z-zx=gE}~$Yn{zV#ex-@aBwr3RLgnwYmGlykl$%K3yGw>F&r{~b5-Fyoq6z(|Hf`&F zS4D)1w}Sr3=!z;JpSK*>&w-#7(^K^IrZ!qL*}55oVzVk|QJ*}Lk_FGoha!<|Mvu~O zG;E_^xo5*d!Lj%{Lm$t^a8-Nm_9hoDph|8C7Fi#YSz3Q)EB)MmTy=Yq#Q~KsE?Lwp z4(N0(wPv_{S8Y%pPeV01!0f2s)?WRKnH1R2@e0+yp4}=7v03tCzduyvy@AF$jDSsfCgdO1byTV4c za2DYXX@fLvwl09CCSfuR4hGDbIG=2Z8$nla1sRx&z!s3Z!tqJJ1`+D%4`Isq{K`ba zm{6cvco&QiGpAyX!PDvJD4HV2;jd_^h;B^EH(+YV;o=cN?jdoB?JM3+^aL&AfH9+I zMsX}$4vJ5RaOG2`kx~Ky2tY~<&H+}}&?k-_8Uu^+dkpodMJEYbX!ZvN#r^uim=MC9 zO&C+Eh7VR(Fs)^vC?i@T0hm}_Sjh?nR7Y0|B-!kzs9I1cNl#Y}hQz7^VexCX_@(Hn zs1nZTRyLE8v5{q@!9%c;p(O&)&9k5&OXZ{yh2a`Jl{%G@N17SMbNji4NMCDdYw|>r?YZo9=5tH->`~kFg!XYr zF9&luZptZR#=%O|CxCnm)GKrGDbK_c3E4JHAqHeko`fG`>ewS{!W^EnZ|<75;0$it zZd}2(?W4bJmg!e_T)SOP05jZn%`;XxlfaJVQ49B|oJ-mSajk$HXHJRx{kpKT*FKep z9%n}zKbyJ6FT#&Iz(r7VF@oFy97BnZfFWo6zaomZE{7spweL^5I`OC@U>#@V1UNlv zfUGQdN7FiJ%g9c}WUhWVW1HDDpD4%0d~i%Kj|f2TYH#eQj?fUk)2uKwTk z+h8%iG=<-VN+n$1->54xDDS!{lYw_;oVIHr+tBuv_<#PL{3$a{lh? zq^`!k^j^)3O&D4xO3%I1!2WrJ>|L#B6Gqlj*h3>k^~SFYI?XUk=GBw-u_E4g=B!f8KJ!2Q6L>Q63RA$zQHeF?%c27KaCmPS#M|&(~yU7fCD4zEdmiR8tfd zELy=|)SZ8*gCd?Ni>RyVovB8#1<@JDh%Z_&P}H5jpe+G++RlMO&(TKG#lG?q*$z72 zabvFn=<+zZh~hZ3>S_TuHb1QUm1ulg3BZ~yIfQi$gI(TyZ%r8w4zhP&C+!X0UAxog zp?U=JlKvhLewbeV<3QJi}y!eN81z^pk(-V{iVP)Y0|t ztdB$znh8tFRfSSs9FuW%NlzO<|YWIy}1KG9nH#U58lGXT}XYF;OmY`6)+w(gz! ze=TVB-fY{NE+(%EE!)y90W4bk?1ocKa>|9fe~ij~k{%U;p!Z z_I@Ke4?M7mUD`xevwN$ga#s61Sw%x%43b{<`n#l|Hk8qmIW`S-37n1x#hkB+$5 z6`mgtJ0qsf^(eR=?gpaF+-u7@My_;b&%a0p=xYx)YZF$g6Rv&aEL@8sKJl1jiHGTN ziN_qvkL9e<>vtuXfC&f5GWcfl)1>uNr>xw6We0F;!ZCLQ-IBu$DT&Uoxc+L$KXx>qPW+!yL5~de&s5oVn z#&m9rq$)T0`2VmH85UNTmv>xA5)@>XoRris)gEeM8*e1FYh;n!(W9*L zDCsBN?9g{e`}%@8A`(e2f4kLZ!GWDSt0UoS5AgQip&>!B!<=XLeTN)bXWLkuc-h65 z$)*P`leHCmfJ4!uk=A~xjJ>jfRM02BSY96LQGH$Ptuf+T{Nc;wzm{j@sjh#k6_-8?-k4&gVuNEU3`hOOQ*x^@MC=$Q&pp_*)?*X4r-}rG`P;%{ z)y4UJ9<2tRI?1N`bRuiYlNvl1IhpAUDLkMN`f7tm+xAp>3_%cKMIs=T_s`}4SK}AB zBKgs2m*QX_G zR^<_@)9jLJHG6-ItY;w5TK3Gdq=7y01GpsIL8}syUPI&;0r~(V-vULr!?v~qab2F$ zb4I$Q6UBquN)5IPF2vB!@=fd5J+I*muRa4!{KG#XRqTmx!CkF;rQaKl@O7;UbQQcDsaFxf%+KQC9rD09oA7XMgo0z{_L3)&dL_Bqg{I z-dt-1JuxgY9c=DJvXQf785bFil{uDln6hxi6gbaIb1bvs^A~})GcSS3bIay9mLnWI z+jKhY_$iLj73${zT85GP9Y!wntClcw(Tcm4HO`a<+tHGRs^+HAUWAg^^VMXV>u`wZ z!bi8n285=*V+Bj>pmtVAqzZ9s9%$y4iU>|?Xt1xvkAWzuCL>;pA|Q*qz`RXFxolU` z=h6kwgvuXHua@0)-_&7_d z>{~CA9&X_lLvep$cJ8Wh-0LLsWXE9_JT43f_JiZZtaMFkZsW7fZ;)FTulskIyP(*0 zKb{5X=JJ;@0zqd}pfA(1A!5sE9^~>NRaRIThip!9lcO%K3w3eViAxlg%^)E3iyO#L z1yI1v&y<$b*~lnqDrmT4l)T1Tu0TQ<-^7nMpC@Z7l|&pF57mdf+DUroZyMR-Wzy>I z&PY5HNgyTeINfF@5;lX{G#E9b&cvfuEUff&D{9OxmBVMY=M@rT_s(P8S^|&tYmc|F z_YRQN#AJ7_BMN)%I9c6<8Yfk_-6$yR>+kFD>g($54a`5VRtnHd4^^-QNm{)D;Na=L zLO{?bzTL(iUqiOgGyi!NZT@_{Cw8|y@E64A$%?hGLDsY6HxZY^^(xtwUy@4t$-h~> z{I#@SDeTD!&rbMh%fN@~B<0GGXvy}z53vv3u;WwR&hIU_j)WS5=zc?OaC z9&4!;)(JSxjKMYm>C!X*qj7WL#Z9h-@|)OOlH_Ce)=6cqZR9VsjrJ{++S;yDxhhlH ze1ZLPE{iGhbB=H0QnsyK5#>6A9PnwB$K)wj)^{su>nb*`kQJ~~W_FwyzThag@x5kN z`2lHT3MEa*nPuWy%EA!Gb%%9>KT*9n3;Dk@^j#M;vY- zeb*LssfsYn6y&SHpHsRO6j?`Z5m9Z?fB!FPe${*3#`&6ss-#x-{t#K8yV>rdt%J7h zw&K<7A>wX5oUVb(`kHUAGVVXjivWx92`4} z`c%c<@m~&)=3+dLI+egKs;dO5A%0C>@W~!(B?H$Loq`vFi0a`LVdD&9&$g3I^n<6q z)>5~Q!*eg3J$C4r zy#lNe_3GqOUksix!=mYTHsqLqjkl8<`ikxXF=pv2LW2*;=YjeXD==+6=?@fjs0yF@ zFGu(i#vbA2-R_+#2kcmPhR3>j3apL(^6e`64{z3%xg0-GbfBPJAHL#k#Sr@5-;P$= zlSgBE(rB@3KL*B-D#H~npIGb_R8o0hQ#FG!DM$pSYQI?1m`xHO=$;o0C zQx&`G=-XSD>$>~x-sQSpe|r!8>KirH;vDwPKy%cfZEWCH(!NbdPEYu+q!VsZ72~Wo zz^*{UM6`VbJ&{+iCN#xb?iz}{g`UX!DhxfZ_cx^@5ZtYJYa@N_>2kW~h1F|4ameKL zWJ}X%5!QrQ|Md&;5tPgDw+Z?05K0O1<%fMdVcxpJ!Vu`YcNCrN-$tPCEpyWk+%0-?Dr!6hJAq%u@m@bwd%X)a@vP=0atV=dcGKRIjANm&^m&hC4ZwC?96aWenWiMU8wmrEoT zZR21j7PoXa+2HukJeGo}hVU2IBVQ)1)u?X8d%I38p=XdaXU{7rG5F1ITWQAPdLoK# zolQ3u0(-s4@~`3Ngw>d)_x-~rcIC?m!G60A&p_91khb*aW&3$SO)lW8xZszM>X~W0 zmXHsj_D8&e(Gnc4?N{Hcb_=}pH@~vF4#E1q^Y&JD*(H0flWcYK=I!jcAAmX#kzQB7 zWLGC~&Yswe>VkzIS8NrP2}Nf;X2^n9K?y1kxUfqHNfWE-BQ0`7l%mFTJOtojDkmtz z%TMimI1N40mWyvxDBR+TRLYSSK7|(52qY*KKl!@wSq_=6^=UZKOwZag-DR3C;XXiTM zPd~hy)HEf`&JYI#G9V)03zy+2s=7ojviEv~u#*a@_AP@|ub$ULK3}B|KeL9He%%z| z4+R~9fnPtZnQ$|B-Bmm?D$>=87}4qWIUYzn3fI?p8HZZf)9i3Pa-jtPs7ATX7Jhxd z&sjefQrvUfg-tH<1;=-==X%LH%=As^L_kS{{hn9akih+pAW8LWo5wlNtNoc!E> zM|AKfzfUTvZBCC_R@w~j-09Z$F>e4*et~27fRg?00_V?PWdRH*xqy>To|%~mF5TVf zM3F^u`OcJXE$ytZ&mA-}QwqXB8xDAsrtF{O>5uI|om-EBI(ZEsd#=HQrt1P^C+=h*{_<1=y4$5~X)<9$8qWCmxwA6x(Lg);ys233xkqF`JIt600V^2avh!+c-mrAYA+Y9ypyj+gM!v-8% z&}%$}FG(DOp9!V=ab8WY%FbaB-LbEtMYS}_i%{#ypC;Rr?3buQMm9JcuH1&wc9Hsm z2Ehw}qg-HNGmbv-nO|1rctT=F1dljNPGoqxQqHEi#56d3qo(W^&Z!v?2jdwd@suA2 z^*a!-d$3dr2mzTrQ#PTBf zgoAk^nNuQgYqh-C3*fiI?Okr+V7@5rH?Cn@<=hGPIMZwI+R64TE zku@e4Zo$%B?#bB5L6Ly7o4e}q$9fK`Ty5CQluw)JV!=pE-a7`J#S zg6tj#ks~jht zTeoaRfb4+^<|ejo!X6S2h>Fk7k8T6Jn=RBy>)A8C$b@sl8;UY#c=MW&LNn90#P)dO zujQ8S3+%)1BabB4A>Z=3_0p=vSJq4Iq+A@8y;Co3V}~9k+quKAihgc!4cW~-hBmgR zlT?>+V~RaeFL}r;wWsS z9q!P>w-DbdfE|AagTD3idI619w~4f$H(&>Ate19{?L$<*Cjvb5FN$L{SUzJ>|oJV!&XW9onqPytx@G6+fI zH;1|g&Fiv*fh4QHMYnF|;EN%qJ%woWd-I4=E{ zRoFUpo79Y7{u7dK@zL9)C_i1*3F!~lCW8kKjqKe&b`yK=1pXM(+b1OOO>p1A@na{3 zx>T=sD1^^@4~^{`>Q)-Bq)HSxcK3(a#ov~O*)w-Z4S%#CC8TmO WpPjoXZMR<|@&1v;#6{_X^#1@qICXFU delta 902 zcmY+DU1$?o6vw%9l4>=ves*HJ)ihw8ks&0lHD8WaTd6_QWUJurgG!oAY+`2SI+++_ z6S1-=_<_PzrXYTyKB-U$G7qBb+deGr!?Ntd!a{u$L6IT|yHq^!v%Z{r&-tHw?vMZF zzW&Ai;>@B;r=1&nL;XRv^iVHr`^nXqiaZIB0^3)J63X0;EK%9JLY<7VHx7YR+4aMH zq{0$TjSRAFMK7U^f1P_cl48MyBiMR!;VV3*wI+$$c+z@`w5qYAX~UbBDf{Eder%A8 zb1kYE)uIA>e}rdA*LE!OrE?_C*0@u6v5s-{_ER@;r|R`wS<9GtUGt{cvn3%?ZJaIN zAV{&HA}y~7gMMo0rfycLB1&0dkd0Tuh;^&7731@Y-dPci8cdgd@{tFN`hg|sNC zY+1)mWDiZ2(V_>FLnx}9CmQtB<6$&55Zs8y_MDkX5PK%U-8gMe|Ha*N0hLtyz;o{J zrG#4jv)lkKM`yXcgxbH)a=#IxTI?;io~ZVtw;Wqc=#?bA!QudcZfqSRP-v%uDN0JM zHc=Prihp}iAM5E6RBMjFYWsfz7o01o?DNHPx*?S0K&-U6922Q^bpdR_TmD%9uYglU zYt0gy@Sj9N{gmgt7{*!QCw?zD7JL7d3vgl@qc&zHP;^d_xk6g}sq z*7PdqwgW36+ho473LYZfu68&dqW&F;)ZU-vfJFVfliP;kBg1kK_e05%9da*jzCN8_ zF$~?1wQ{j2QELego{ssdj?XqvAK-zHe6JVvbXL)_pC`;4tZPjPX$|ws_)9!=BI11> z)*J_C6Uz&Ys;0HSPmy C#V%?9 diff --git a/public/mix-manifest.json b/public/mix-manifest.json index beeaf8368d9029087ec62346259a2d8d4c487631..2675c6ea988a57507c80dffbed7e10ccb7038757 100644 GIT binary patch delta 45 zcmcbqcT;aeGpj + +@endsection + +@push('scripts') + +@endpush diff --git a/resources/views/admin/settings/sidebar.blade.php b/resources/views/admin/settings/sidebar.blade.php index b6f83bd04..be411198e 100644 --- a/resources/views/admin/settings/sidebar.blade.php +++ b/resources/views/admin/settings/sidebar.blade.php @@ -1,31 +1,76 @@ -@section('menu') -
    -
  • - Home -
    • -{{--
  • - Backups -
    • --}} - {{--
  • - Configuration -
    • --}} - {{--
  • - Customize -
    • --}} - {{--
  • - Features -
    • --}} - {{--
  • - Maintenance -
    • --}} -
  • - Pages -
    • - {{--
  • - Storage -
    • --}} -
  • - System -
    • +
    + -@endsection +
    diff --git a/routes/web.php b/routes/web.php index 73bc7c24c..7a9352201 100644 --- a/routes/web.php +++ b/routes/web.php @@ -94,6 +94,8 @@ Route::domain(config('pixelfed.domain.admin'))->prefix('i/admin')->group(functio Route::get('directory/home', 'AdminController@directoryHome')->name('admin.directory'); + Route::get('autospam/home', 'AdminController@autospamHome')->name('admin.autospam'); + Route::prefix('api')->group(function() { Route::get('stats', 'AdminController@getStats'); Route::get('accounts', 'AdminController@getAccounts'); @@ -129,6 +131,17 @@ Route::domain(config('pixelfed.domain.admin'))->prefix('i/admin')->group(functio Route::get('reports/spam/all', 'AdminController@reportsApiSpamAll'); Route::get('reports/spam/get/{id}', 'AdminController@reportsApiSpamGet'); Route::post('reports/spam/handle', 'AdminController@reportsApiSpamHandle'); + Route::post('autospam/config', 'AdminController@getAutospamConfigApi'); + Route::post('autospam/reports/closed', 'AdminController@getAutospamReportsClosedApi'); + Route::post('autospam/train', 'AdminController@postAutospamTrainSpamApi'); + Route::post('autospam/search/non-spam', 'AdminController@postAutospamTrainNonSpamSearchApi'); + Route::post('autospam/train/non-spam', 'AdminController@postAutospamTrainNonSpamSubmitApi'); + Route::post('autospam/tokens/custom', 'AdminController@getAutospamCustomTokensApi'); + Route::post('autospam/tokens/store', 'AdminController@saveNewAutospamCustomTokensApi'); + Route::post('autospam/tokens/update', 'AdminController@updateAutospamCustomTokensApi'); + Route::post('autospam/tokens/export', 'AdminController@exportAutospamCustomTokensApi'); + Route::post('autospam/config/enable', 'AdminController@enableAutospamApi'); + Route::post('autospam/config/disable', 'AdminController@disableAutospamApi'); }); }); diff --git a/storage/app/.gitignore b/storage/app/.gitignore index 693e05ed8..40f532b85 100755 --- a/storage/app/.gitignore +++ b/storage/app/.gitignore @@ -2,5 +2,6 @@ !backups/ !public/ !remcache/ +!nlp/ !cities.json !.gitignore diff --git a/storage/app/nlp/.gitignore b/storage/app/nlp/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/storage/app/nlp/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore