From 863fb701ad21b80e3223500701cbb1628bae77b9 Mon Sep 17 00:00:00 2001 From: Andrii Nikitin Date: Tue, 14 Nov 2023 11:44:39 +0100 Subject: [PATCH] Add diagnostic job for checking presence of a file on mirrors --- assets/javascripts/mirrorlist.js | 15 +++ lib/MirrorCache/Task/MirrorFileCheck.pm | 106 ++++++++++++++++++ lib/MirrorCache/WebAPI.pm | 1 + .../WebAPI/Controller/Rest/ServerCheckFile.pm | 39 +++++++ lib/MirrorCache/WebAPI/Plugin/Backstage.pm | 4 +- templates/mirrorlist.html.ep | 5 + 6 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 lib/MirrorCache/Task/MirrorFileCheck.pm create mode 100644 lib/MirrorCache/WebAPI/Controller/Rest/ServerCheckFile.pm diff --git a/assets/javascripts/mirrorlist.js b/assets/javascripts/mirrorlist.js index ec2362a6..651db2b0 100644 --- a/assets/javascripts/mirrorlist.js +++ b/assets/javascripts/mirrorlist.js @@ -191,3 +191,18 @@ function initMap(lat, lng, idx) { // setTimeout(function(){ marker.showPopup(); }, 500); } +function checkFileOnMirrors(path) { + $.ajax({ + url: '/rest/server/check_file?file=' + path, + type: "PUT", + dataType: 'json', + success: function(response) { + handleCheckFileOnMirror(response.job_id); + }, + error: handleAdminTableApiError + }); +} + +function handleCheckFileOnMirror(job_id) { + $(location).attr('href', '/minion/jobs?id=' + job_id); +} diff --git a/lib/MirrorCache/Task/MirrorFileCheck.pm b/lib/MirrorCache/Task/MirrorFileCheck.pm new file mode 100644 index 00000000..1b5b3249 --- /dev/null +++ b/lib/MirrorCache/Task/MirrorFileCheck.pm @@ -0,0 +1,106 @@ +# Copyright (C) 2023 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package MirrorCache::Task::MirrorFileCheck; +use Mojo::Base 'Mojolicious::Plugin'; + +use Mojo::UserAgent; + +sub register { + my ($self, $app) = @_; + + $app->minion->add_task(mirror_file_check => sub { _check($app, @_) }); +} + +sub _check { + my ($app, $job, $path, $args) = (shift, shift, shift, ref $_[0] ? $_[0] : {@_});; + return $job->fail('Empty path is not allowed') unless $path; + + my $minion = $app->minion; + my $schema = $app->schema; + + # my $mirrors = $schema->resultset('Server')->search({country => 'de'}); + my @mirrors = $schema->resultset('Server')->search({ enabled => '1' }); + my ($countok, $counterr, $countoth) = (0, 0, 0); + my $concurrency = 8; + my $current = 0; + $concurrency = @mirrors unless $concurrency < @mirrors; + $job->note(_concurrency => $concurrency); + + for (my $i = 0; $i < $concurrency; $i++){ + my $m = shift @mirrors; + next unless $m; + my $id = $m->id; + my $urldir = $m->urldir; + $urldir = '/' unless $urldir; + my $url = $m->hostname . $m->urldir . $path; + # it looks that defining $ua outside the loop greatly increases overal memory usage footprint for the task + my $ua = Mojo::UserAgent->new->request_timeout(4)->connect_timeout(4); + + my ($next, $then, $catch); + my $started = time(); + $current++; + + + $next = sub { + $m = shift; + unless ($m) { + $current--; + unless ($current) { + Mojo::IOLoop->stop unless $current; + } + return; + }; + $id = $m->id; + my $urldir = $m->urldir; + $urldir = '/' unless $urldir; + $url = $m->hostname . $m->urldir . $path; + $started = time(); + return $ua->head_p($url)->then($then, $catch); + }; + + $then = sub { + my $tx = shift; + my $elapsed = int(1000*(time() - $started)); + my $code = $tx->res->code; + $job->note($m->hostname => $code . " ($elapsed ms) id=$id " . $tx->req->url); + if ($code == 200) { + $countok++; + } elsif ($code > 399) { + $counterr++; + } else { + $countoth++; + } + $next->(shift @mirrors); + }; + + $catch = sub { + my $err = shift; + my $elapsed = int(1000*(time() - $started)); + $job->note($m->hostname => $err . " ($elapsed ms) id=$id " . $url); + $counterr++; + $next->(shift @mirrors); + }; + + $ua->head_p($url)->then($then, $catch); + } + # sleep 5; + + Mojo::IOLoop->start; + $job->note(_ok => $countok, _err => $counterr, _oth => $countoth); + $job->finish; +} + +1; diff --git a/lib/MirrorCache/WebAPI.pm b/lib/MirrorCache/WebAPI.pm index 18cca253..4f838716 100644 --- a/lib/MirrorCache/WebAPI.pm +++ b/lib/MirrorCache/WebAPI.pm @@ -191,6 +191,7 @@ sub _setup_webui { $rest_operator_r->post('/server/:id')->name('post_server')->to('table#update', table => 'Server'); $rest_operator_r->delete('/server/:id')->to('table#destroy', table => 'Server'); $rest_operator_r->put('/server/location/:id')->name('rest_put_server_location')->to('server_location#update_location'); + $rest_operator_r->put('/server/check_file')->name('rest_put_server_check_file')->to('server_check_file#start'); $rest_operator_r->post('/server/note/#hostname')->name('rest_put_server_note')->to('server_note#ins'); $rest_operator_r->get('/server/note/#hostname')->name('rest_get_server_note')->to('server_note#list'); $rest_operator_r->get('/server/contact/#hostname')->name('rest_get_server_contact')->to('server_note#list_contact'); diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/ServerCheckFile.pm b/lib/MirrorCache/WebAPI/Controller/Rest/ServerCheckFile.pm new file mode 100644 index 00000000..620d7174 --- /dev/null +++ b/lib/MirrorCache/WebAPI/Controller/Rest/ServerCheckFile.pm @@ -0,0 +1,39 @@ +# Copyright (C) 2023 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package MirrorCache::WebAPI::Controller::Rest::ServerCheckFile; +use Mojo::Base 'Mojolicious::Controller'; +use Data::Dumper; + +sub start { + my ($self) = @_; + + my $path = $self->param("file"); + return $self->render(code => 400, text => "Mandatory argument is missing") unless $path; + + my $job_id; + eval { + $job_id = $self->minion->enqueue('mirror_file_check' => [$path]); + }; + return $self->render(code => 500, text => Dumper($@)) unless $job_id; + + return $self->render( + json => { + job_id => $job_id, + } + ); +} + +1; diff --git a/lib/MirrorCache/WebAPI/Plugin/Backstage.pm b/lib/MirrorCache/WebAPI/Plugin/Backstage.pm index 127acd38..25aaf381 100644 --- a/lib/MirrorCache/WebAPI/Plugin/Backstage.pm +++ b/lib/MirrorCache/WebAPI/Plugin/Backstage.pm @@ -50,6 +50,7 @@ sub register_tasks { $app->plugin($_) for ( qw(MirrorCache::Task::MirrorCheckFromStat), + qw(MirrorCache::Task::MirrorFileCheck), qw(MirrorCache::Task::MirrorScanScheduleFromMisses), qw(MirrorCache::Task::MirrorScanScheduleFromPathErrors), qw(MirrorCache::Task::MirrorScanSchedule), @@ -91,7 +92,8 @@ eval { $app->minion->backend->no_txn(1) if $db eq 'mysql'; $self->register_tasks; -}; + 1; +} or $app->log->error($@); # Enable the Minion Admin interface under /minion my $auth = $app->routes->under('/minion')->to('session#ensure_operator'); diff --git a/templates/mirrorlist.html.ep b/templates/mirrorlist.html.ep index ceff4c39..5189fd68 100644 --- a/templates/mirrorlist.html.ep +++ b/templates/mirrorlist.html.ep @@ -19,6 +19,7 @@ img.huechange1 { filter: hue-rotate(90deg) }