Centalized mirror management (#417)

* Ignore mirrors from other regions if MIRRORCACHE_REGIONS is configured * Forbid server editing if MIRROR_PROVIDER is configured * Improve geo awareness of Mirror details form * When mirrors are imported using MIRROR_PROVIDER option - refer the provider in "Private notes". * When mirrors are maintaned by subsidiary - refer corresponding subsidiary for "Last incidents" table.
openSUSE · Oct 16, 2023 · ad68cb4 · ad68cb4
1 parent b32713d
commit ad68cb4
Show file tree

Hide file tree

Showing 15 changed files with 314 additions and 40 deletions.
diff --git a/assets/javascripts/admintable.js b/assets/javascripts/admintable.js
@@ -325,7 +325,11 @@ function renderAdminTableActions(data, type, row, meta) {
         return '';
     }
     var url = $("#admintable_api_url").val();
-    res = '<button type="submit" class="btn" alt="Edit" title="Edit" onclick="setEditingAdminTableRow(this.parentElement, true, false);"><i class="far fa-edit"></i></button>';
+    var noActions = document.getElementById("regionmirrorproviderpanel");
+    var res = '';
+    if (! noActions ) {
+        res = '<button type="submit" class="btn" alt="Edit" title="Edit" onclick="setEditingAdminTableRow(this.parentElement, true, false);"><i class="far fa-edit"></i></button>';
+    }
     if (url == '/rest/server' && data) {
         return res + '<button type="submit" class="btn" alt="UpdateLocation" title="Update Location" onclick="submitServerLocationRow(this.parentElement, ' + data + ');"><i class="far fa-bookmark"></i></button>';
     } else {

diff --git a/lib/MirrorCache/Config.pm b/lib/MirrorCache/Config.pm
@@ -48,6 +48,7 @@ has top_folders   => $ENV{MIRRORCACHE_TOP_FOLDERS};
 
 has plugin_status => $ENV{MIRRORCACHE_PLUGIN_STATUS};
 
+has regions               => $ENV{MIRRORCACHE_REGIONS};
 has mirror_provider       => $ENV{MIRRORCACHE_MIRROR_PROVIDER};
 
 has db_provider           => undef;
@@ -64,7 +65,7 @@ sub init($self, $cfgfile) {
     my $cfg;
     $cfg = Config::IniFiles->new(-file => $cfgfile, -fallback => 'default') if $cfgfile;
     if ($cfg) {
-        for my $k (qw/root root_nfs redirect redirect_huge huge_file_size small_file_size city_mmdb ip2location top_folders mirror_provider browser_agent_mask custom_footer_message country_image_dir/) {
+        for my $k (qw/root root_nfs redirect redirect_huge huge_file_size small_file_size city_mmdb ip2location top_folders regions mirror_provider browser_agent_mask custom_footer_message country_image_dir/) {
             if (my $v = $cfg->val('default', $k)) {
                 $self->$k($v);
             }

diff --git a/lib/MirrorCache/Schema.pm b/lib/MirrorCache/Schema.pm
@@ -16,6 +16,8 @@ my $SINGLETONR;
 
 my $PROVIDER;
 
+my $OUR_REGIONS;
+
 sub pg {
     return 1 if $PROVIDER eq 'Pg';
     return 0;
@@ -26,7 +28,7 @@ sub provider {
 }
 
 sub connect_db {
-    my ($self, $provider, $dsn, $user, $pass) = @_;
+    my ($self, $provider, $dsn, $user, $pass, $our_regions) = @_;
 
     $PROVIDER = $provider;
 
@@ -38,6 +40,13 @@ sub connect_db {
         }
 
         $SINGLETON = __PACKAGE__->connect($dsn, $user, $pass);
+
+        if ($our_regions) {
+            my @regions = split ',', $our_regions;
+            my $in = join ', ', map "'$_'", @regions;
+
+            $OUR_REGIONS = "and (s.region in ($in) or (select enabled from server_capability_declaration where server_id = s.id and capability = 'region' and extra in ($in)))"
+        }
     }
 
     return $SINGLETON;
@@ -103,4 +112,10 @@ sub migrate {
     my $db = $conn->db; # this will do migration
 }
 
+sub condition_our_regions {
+    return '' unless $OUR_REGIONS;
+
+    return $OUR_REGIONS;
+}
+
 1;
diff --git a/lib/MirrorCache/Schema/ResultSet/Server.pm b/lib/MirrorCache/Schema/ResultSet/Server.pm
@@ -94,6 +94,7 @@ sub mirrors_query {
     $limit1 = $limit + $limit if $limit > 10;
 
     my $join_server_project = "";
+    my $condition_our_regions = $schema->condition_our_regions;
     my $condition_server_project = "";
     if ($project_id) {
         $join_server_project = "left join server_project sp on (project_id,sp.server_id) = ($project_id,s.id) and state < 1";
@@ -145,7 +146,7 @@ from (
     from folder_diff fd
     join file fl on $join_file_cond
     join folder_diff_server fds on fd.id = fds.folder_diff_id and date_trunc('second', fl.dt) <= fds.dt
-    join server s on fds.server_id = s.id and s.enabled  $country_condition
+    join server s on fds.server_id = s.id and s.enabled  $country_condition $condition_our_regions
     left join server_capability_declaration scd on s.id = scd.server_id and scd.capability = 'country'
     left join folder_diff_file fdf on fdf.file_id = fl.id and fdf.folder_diff_id = fd.id
     $join_server_project
@@ -200,8 +201,9 @@ sub folder {
     my $rsource = $self->result_source;
     my $schema  = $rsource->schema;
     my $dbh     = $schema->storage->dbh;
+    my $condition_our_regions = $schema->condition_our_regions;
 
-    my $sql = <<'END_SQL';
+    my $sql = <<"END_SQL";
 select s.id as server_id,
 concat(
     case
@@ -227,6 +229,7 @@ where
 (fds.folder_diff_id IS NOT DISTINCT FROM fd.id OR fds.server_id is null)
 AND (cap_fhttp.server_id IS NULL or cap_fhttps.server_id IS NULL)
 AND (sp.server_id IS NULL)
+$condition_our_regions
 group by s.id, s.hostname, s.urldir, f.path, cap_http.server_id, cap_fhttp.server_id, cap_hasall.capability
 order by s.id
 END_SQL
@@ -246,8 +249,9 @@ sub server_projects {
     my $rsource = $self->result_source;
     my $schema  = $rsource->schema;
     my $dbh     = $schema->storage->dbh;
+    my $condition_our_regions = $schema->condition_our_regions;
 
-    my $sql = <<'END_SQL';
+    my $sql = <<"END_SQL";
 select concat(s.id, '::', p.id) as _key,
         s.id as server_id,
         p.id as project_id,
@@ -258,7 +262,7 @@ from project p
     join server s on s.enabled
     left join server_project sp on sp.server_id = s.id and sp.project_id = p.id
 where
-    coalesce(sp.state,0) > -1
+    coalesce(sp.state,0) > -1 $condition_our_regions
 END_SQL
     return $dbh->selectall_hashref($sql, '_key', {});
 }

diff --git a/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm b/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
@@ -26,11 +26,14 @@ sub search_by_country {
     my $schema  = $rsource->schema;
     my $dbh     = $schema->storage->dbh;
 
-    my $sql = <<'END_SQL';
+    my $condition_our_regions = $schema->condition_our_regions;
+
+    my $sql = <<"END_SQL";
 select concat(CASE WHEN length(s.hostname_vpn)>0 THEN s.hostname_vpn ELSE s.hostname END,s.urldir,'/') as uri, CASE WHEN length(s.hostname_vpn)>0 THEN s.hostname_vpn ELSE s.hostname END as hostname, s.id as id,
-    -- server has capability enabled when two conditions are true:
+    -- server has capability enabled when following conditions are true:
     -- 1. server_id is not mentioned in server_capability_force
     -- 2. there is no entry in server_capability_declaration which has enabled='F' for the server_id.
+    -- 3. additional string as provided by schema->condition_our_regions;
     COALESCE(fhttp.server_id  = 0, COALESCE(http.enabled,'t'))  as http,
     COALESCE(fhttps.server_id = 0, COALESCE(https.enabled,'t')) as https,
     COALESCE(fipv4.server_id  = 0, COALESCE(ipv4.enabled,'t'))  as ipv4,
@@ -71,7 +74,7 @@ select concat(CASE WHEN length(s.hostname_vpn)>0 THEN s.hostname_vpn ELSE s.host
     where
         (fhttp.server_id IS NULL or fhttps.server_id IS NULL) -- do not show servers which have both http and https force disabled
     AND (fipv4.server_id IS NULL or fipv6.server_id IS NULL)  -- do not show servers which have both ipv4 and ipv6 force disabled
-    AND s.enabled
+    AND s.enabled $condition_our_regions
 END_SQL
 
     unless ($dbh->{Driver}->{Name} eq 'Pg') {

diff --git a/lib/MirrorCache/WebAPI.pm b/lib/MirrorCache/WebAPI.pm
@@ -76,7 +76,7 @@ sub startup {
     my $db_provider = $mcconfig->db_provider;
 
     eval {
-        MirrorCache::Schema->connect_db     ($mcconfig->db_provider, $mcconfig->dsn,         $mcconfig->dbuser, $mcconfig->dbpass);
+        MirrorCache::Schema->connect_db     ($mcconfig->db_provider, $mcconfig->dsn,         $mcconfig->dbuser, $mcconfig->dbpass, $mcconfig->regions);
         MirrorCache::Schema->connect_replica($mcconfig->db_provider, $mcconfig->dsn_replica, $mcconfig->dbuser, $mcconfig->dbpass) if $mcconfig->dsn_replica;
         1;
     } or warn("Database connect failed: $@");

diff --git a/lib/MirrorCache/WebAPI/Controller/App/Myserver.pm b/lib/MirrorCache/WebAPI/Controller/App/Myserver.pm
@@ -17,7 +17,17 @@ package MirrorCache::WebAPI::Controller::App::Myserver;
 use Mojo::Base 'MirrorCache::WebAPI::Controller::App::Table';
 
 sub index {
-    shift->SUPER::admintable('myserver');
+    my $c = shift;
+    my $mirror_provider = $c->mcconfig->mirror_provider;
+    if (my $url = $c->mcconfig->mirror_provider) {
+        $url =~ s!^https?://(?:www\.)?!!i;
+        $url =~ s!/.*!!;
+        $url =~ s/[\?\#\:].*//;
+        my $mirror_provider_url = 'https://' . $url . '/app/server';
+        $c->stash( mirror_provider_url => $mirror_provider_url );
+    }
+
+    $c->SUPER::admintable('myserver');
 }
 
 sub update {

diff --git a/lib/MirrorCache/WebAPI/Controller/App/Server.pm b/lib/MirrorCache/WebAPI/Controller/App/Server.pm
@@ -17,7 +17,17 @@ package MirrorCache::WebAPI::Controller::App::Server;
 use Mojo::Base 'MirrorCache::WebAPI::Controller::App::Table';
 
 sub index {
-    shift->SUPER::admintable('server');
+    my $c = shift;
+    my $mirror_provider = $c->mcconfig->mirror_provider;
+    if (my $url = $c->mcconfig->mirror_provider) {
+        $url =~ s!^https?://(?:www\.)?!!i;
+        $url =~ s!/.*!!;
+        $url =~ s/[\?\#\:].*//;
+        my $mirror_provider_url = 'https://' . $url . '/app/server';
+        $c->stash( mirror_provider_url => $mirror_provider_url );
+    }
+
+    $c->SUPER::admintable('server');
 }
 
 sub update {
@@ -49,12 +59,26 @@ sub show {
     if ($self->is_operator) {
         $admin_email = $self->schema->storage->dbh->selectrow_array("SELECT msg FROM server_note WHERE hostname = ? AND kind = 'Email' ORDER BY dt DESC LIMIT 1", undef, $hostname);
     }
+    my $subsidiary;
+    if (my $regions = $self->mcconfig->regions) {
+        if ($f->region && -1 == CORE::index($regions, $f->region)) {
+            $subsidiary = $self->subsidiary->url($f->region);
+        }
+    }
+    my $provider;
+    if ($provider = $self->mcconfig->mirror_provider) {
+        $provider =~ s!^https?://(?:www\.)?!!i;
+        $provider =~ s!/.*!!;
+        $provider =~ s/[\?\#\:].*//;
+    }
 
     my $server = {
         id           => $f->id,
         hostname     => $f->hostname,
         public_notes => $f->public_notes,
         admin_email  => $admin_email,
+        subsidiary   => $subsidiary,
+        provider     => $provider,
     };
 
     return $self->render('app/server/show', server => $server);

diff --git a/t/environ/01-smoke-mirror-provider.sh b/t/environ/01-smoke-mirror-provider.sh
@@ -8,9 +8,6 @@ mcnaeast=$(environ mc2 $(pwd))
 ap8=$(environ ap8)
 ap7=$(environ ap7)
 
-$mcmirror/gen_env MIRRORCACHE_MODE=mirror_provider
-
-
 $mcmirror/start
 
 $mcmirror/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"

diff --git a/t/environ/09-stability-force2.sh b/t/environ/09-stability-force2.sh
@@ -3,13 +3,16 @@ set -ex
 
 mc=$(environ mc $(pwd))
 
+$mc/gen_env MIRRORCACHE_REGIONS=sa,us-west # when REGIONS is specified - ap9 mirror from another region will be ignored
+
 $mc/start
 $mc/status
 
+ap9=$(environ ap9)
 ap8=$(environ ap8)
 ap7=$(environ ap7)
 
-for x in $mc $ap7 $ap8; do
+for x in $mc $ap7 $ap8 $ap9; do
     mkdir -p $x/dt/{folder1,folder2,folder3}
     echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch
 done
@@ -20,28 +23,40 @@ $ap7/curl /folder1/ | grep file1.1.dat
 $ap8/start
 $ap8/curl /folder1/ | grep file1.1.dat
 
-$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
+$ap9/start
+$ap9/curl /folder1/ | grep file1.1.dat
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','br','sa'"
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap9/print_address)','','t','de','eu'"
+
+$mc/sql "insert into server_capability_declaration(server_id, capability, enabled, extra) select '2','region','t','us-west'"
 
 $mc/curl -I /download/folder1/file1.1.dat
 
-$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/job -e mirror_probe
 $mc/backstage/job folder_sync_schedule_from_misses
 $mc/backstage/job folder_sync_schedule
 $mc/backstage/shoot
 $mc/backstage/job mirror_scan_schedule
 $mc/backstage/shoot
 
-# check redirection works
+echo check redirection works
 $mc/curl -I /download/folder1/file1.1.dat | grep 302
+echo no eu mirror in metalink
+rc=0
+$mc/curl /download/folder1/file1.1.dat.meta4 | grep $($ap9/print_address) || rc=$?
+test $rc -gt 0
 
 # now shut down ap7 and do probe
 $ap7/stop
-$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/job -e mirror_probe
 $mc/backstage/shoot
 
 # check that ap7 is marked correspondingly in server_capability_check
-test 1 == $($mc/db/sql "select count(*) from server_capability_check where server_id=1 and capability='http'")
+$mc/sql_test 1 == "select count(*) from server_capability_check where server_id=1 and capability='http'"
+# echo ap9 is from europe so will not be checked
+$mc/sql_test 0 == "select count(*) from server_capability_check where server_id=3"
 
 # add 4 more failures from the past into DB
 $mc/sql "insert into server_capability_check(server_id, capability, dt) select 1, 'http', min(dt) - interval '15 minute'  from server_capability_check"
@@ -51,9 +66,9 @@ $mc/sql "insert into server_capability_check(server_id, capability, dt) select 1
 $mc/sql "insert into server_capability_check(server_id, capability, dt) select 1, 'https', min(dt) - interval '15 minute'  from server_capability_check"
 $mc/sql "insert into server_capability_check(server_id, capability, dt) select 1, 'https', min(dt) - interval '15 minute'  from server_capability_check"
 
-# make sure we added properly
+echo make sure we added properly
 test 5 == $($mc/db/sql "select count(*) from server_capability_check where server_id=1 and capability='http'")
-# we have inserted 2, plus 2 from manual runs of 'mirror_probe' job, plus 1 'mirror_probe' was scheduled from 'mirror_scan'
+echo we have inserted 2, plus 2 from manual runs of 'mirror_probe' job, plus 1 'mirror_probe' was scheduled from 'mirror_scan'
 test 5 == $($mc/db/sql "select count(*) from server_capability_check where server_id=1 and capability='https'")
 
 $mc/backstage/job -e mirror_force_downs
@@ -62,23 +77,30 @@ $mc/backstage/shoot
 test 1 == $($mc/db/sql "select count(*) from server_capability_force where server_id=1 and capability='https'")
 test 1 == $($mc/db/sql "select count(*) from server_capability_force where server_id=1 and capability='http'")
 
-# age entry, so next job will consider it
+echo age entry, so next job will consider it
 $mc/sql "update server_capability_force set dt = dt - interval '3 hour'"
 
-# now start back ap7 and shut down ap8 but ap7 is not redirected, because it is force disabled
+echo now start back ap7 and shut down ap8 but ap7 is not redirected, because it is force disabled
 $ap7/start
 $ap8/stop
-$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/job -e mirror_probe
 $mc/backstage/shoot
 
 rc=0
-$mc/curl -I /download/folder1/file1.1.dat | grep $($ap7/print_address) || rc=$?
+$mc/curl -I /download/folder1/file1.1.dat?COUNTRY=br | grep $($ap7/print_address) || rc=$?
 test $rc -gt 0
 
-# now scan those mirrors which were force disabled
+echo now scan those mirrors which were force disabled
 $mc/backstage/job -e mirror_force_ups
-$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/job -e mirror_probe
 $mc/backstage/shoot
 
-# ap7 now should serve the request
-$mc/curl -I /download/folder1/file1.1.dat | grep $($ap7/print_address)
+echo ap7 now should serve the request
+$mc/curl -I /download/folder1/file1.1.dat?COUNTRY=br | grep $($ap7/print_address)
+
+echo still no eu mirror in metalink
+rc=0
+$mc/curl /download/folder1/file1.1.dat.meta4 | grep $($ap9/print_address) || rc=$?
+test $rc -gt 0
+
+echo success