Add rollout tracking for iso and repo projects (#451)

lib/Directory/Scanner/OBSReleaseInfo.pm

@@ -0,0 +1,37 @@
+package Directory::Scanner::OBSReleaseInfo;
+
+use Mojo::Base -base, -signatures;
+
+has ['type', 'version', 'versionfilename', 'versionmtime'];
+
+sub new($class, $type, $mtime) {
+    my $self = Mojo::Base::new($class);
+    $mtime = 0 unless $mtime;
+    $self->type($type);
+    $self->versionmtime($mtime);
+    $self->versionfilename('');
+}
+
+sub next_file($self, $filename, $mtime) {
+    return 0 unless $mtime;
+    return 0 if $self->versionmtime > $mtime;
+    if ($self->type eq 'iso') {
+        if ($filename =~ /.*(Build|Snapshot)((\d)+\.?(\d*))-Media\.iso$/) {
+            $self->version($2);
+            $self->versionfilename($filename);
+            $self->versionmtime($mtime);
+            return 1;
+        }
+    } elsif ($self->type eq 'repo') {
+        if ($filename =~ /.*primary.xml(\.(gz|zst))$/) {
+            my $version = $mtime;
+            $self->version($version);
+            $self->versionfilename($filename);
+            $self->versionmtime($mtime);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+1;

lib/MirrorCache/Schema/Result/ProjectRollout.pm

@@ -0,0 +1,30 @@
+use utf8;
+package MirrorCache::Schema::Result::ProjectRollout;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+use DBIx::Class::Timestamps;
+
+__PACKAGE__->table("project_rollout");
+
+
+__PACKAGE__->load_components(qw(InflateColumn::DateTime DynamicDefault));
+__PACKAGE__->add_columns(
+  "project_id",
+  { data_type => "integer"},
+  "epc",
+  { data_type => "integer"},
+  "dt",
+  {
+        data_type   => 'timestamp',
+        is_nullable => 1
+  },
+  version => {
+        data_type   => 'varchar',
+        size => 32
+  },
+);
+
+1;

lib/MirrorCache/Schema/ResultSet/Folder.pm

@@ -33,7 +33,8 @@ where f.sync_requested < (select sync_requested from folder f1 where path = ?) a
 END_SQL
     my $prep = $dbh->prepare($sql);
     $prep->execute($path);
-    return $dbh->selectrow_array($prep);
+    my @res = $dbh->selectrow_array($prep);
+    return \@res;
 }
 
 sub set_wanted {
@@ -403,4 +404,20 @@ END_SQL
     $prep->execute($pathfrom, $pathto, $pathto);
 }
 
+sub add_rollout {
+    my ($self, $project_id, $epc, $version, $versionfile) = @_;
+
+    my $rsource = $self->result_source;
+    my $schema  = $rsource->schema;
+    my $dbh     = $schema->storage->dbh;
+
+    my $sql;
+    $sql = <<'END_SQL';
+insert into project_rollout(project_id, epc, version, filename, dt)
+values (?, ?, ?, ?, now())
+END_SQL
+    my $prep = $dbh->prepare($sql);
+    $prep->execute($project_id, $epc, $version, $versionfile);
+}
+
 1;

lib/MirrorCache/Schema/ResultSet/ProjectRollout.pm

@@ -0,0 +1,71 @@
+use utf8;
+package MirrorCache::Schema::ResultSet::ProjectRollout;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::ResultSet';
+use Mojo::File qw(path);
+
+
+sub project_for_folder {
+    my ($self, $path) = @_;
+    my $dbh = $self->result_source->schema->storage->dbh;
+
+    my $sql = <<"END_SQL";
+select id, name, path, max(epc) as prev_epc,
+  case
+    when upper(name) like '%ISO%' then 'iso'
+    when upper(name) like '%REPO%' and ?::text = concat(path,'/repodata') then 'repo'
+    else ''
+  end as type
+from project
+left join project_rollout on id = project_id
+where ? like concat(path, '/%')
+group by id, name, path
+END_SQL
+    $sql =~ s/::text//g unless ($dbh->{Driver}->{Name} eq 'Pg');
+
+    my $prep = $dbh->prepare($sql);
+    $prep->execute($path, $path);
+    return $dbh->selectrow_hashref($prep);
+}
+
+# the same as project_for_folder, but also profides filename to track
+sub rollout_file_for_folder {
+    my ($self, $path) = @_;
+    my $dbh = $self->result_source->schema->storage->dbh;
+
+    my $sql = <<"END_SQL";
+select id, name, path, epc, filename
+from (
+select id, name, path, max(epc) as prev_epc,
+  case
+    when upper(name) like '%ISO%' then 'iso'
+    when upper(name) like '%REPO%' and ?::text = concat(path,'/repodata') then 'repo'
+    else ''
+  end as type
+from project
+join project_rollout on id = project_id
+where ? like concat(path, '/%')
+group by id, name, path
+) rollout
+join project_rollout on (id, prev_epc) = (project_id, epc) and type != ''
+END_SQL
+    $sql =~ s/::text//g unless ($dbh->{Driver}->{Name} eq 'Pg');
+
+    my $prep = $dbh->prepare($sql);
+    $prep->execute($path, $path);
+    return $dbh->selectrow_hashref($prep);
+}
+
+sub add_rollout_server {
+    my ($self, $server_id, $proj_id, $epc) = @_;
+    my $dbh = $self->result_source->schema->storage->dbh;
+
+    my $sql = 'insert into project_rollout_server(server_id, project_id, epc, dt) select ?, ?, ?, now()';
+    $dbh->prepare($sql)->execute($server_id, $proj_id, $epc);
+    return 1;
+}
+
+1;

lib/MirrorCache/Task/FolderSync.pm

@@ -17,6 +17,8 @@ package MirrorCache::Task::FolderSync;
 use Mojo::Base 'Mojolicious::Plugin';
 use MirrorCache::Utils 'datetime_now';
 
+use Directory::Scanner::OBSReleaseInfo;
+
 my $HASHES_COLLECT = $ENV{MIRRORCACHE_HASHES_COLLECT} // 0;
 my $HASHES_IMPORT  = $ENV{MIRRORCACHE_HASHES_IMPORT} // 0;
 my $HASHES_QUEUE   = $ENV{MIRRORCACHE_HASHES_QUEUE} // 'hashes';
@@ -47,6 +49,11 @@ sub _sync {
 
         $schema->resultset('Folder')->add_redirect($path, $realpath);
     }
+    my $proj = $schema->resultset('ProjectRollout')->project_for_folder($path);
+    my $proj_type;
+    $proj_type = $proj->{type} if $proj;
+    $job->note(proj_id => $proj->{id}, proj_name=> $proj->{name}, proj_prev_epc => $proj->{prev_epc}, proj_type => $proj_type) if $proj_type;
+    my $obsrelease = Directory::Scanner::OBSReleaseInfo->new($proj_type, $proj->{prev_epc}) if $proj_type;
 
     my $folder = $schema->resultset('Folder')->find({path => $realpath});
     unless ($root->is_dir($realpath)) {
@@ -100,6 +107,7 @@ sub _sync {
             $file = $file . '/' if $mmode && $root->is_remote && $mmode < 1000;
             $count = $count+1;
             $schema->resultset('File')->create({folder_id => $folder->id, name => $file, size => $size, mtime => $mtime, target => $target});
+            $obsrelease->next_file($file, $mtime) if $obsrelease;
             return undef;
         };
         eval {
@@ -125,6 +133,7 @@ sub _sync {
             $minion->enqueue('folder_hashes_import' => [$realpath] => {queue => $HASHES_QUEUE}) if $HASHES_IMPORT && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_import', $HASHES_QUEUE);
         }
         $schema->resultset('Folder')->request_scan($otherFolder->id) if $otherFolder && ($count || !$otherFolder->scan_requested);
+        $schema->resultset('Folder')->add_rollout($proj->{id}, $obsrelease->versionmtime, $obsrelease->version, $obsrelease->versionfilename) if $obsrelease && $obsrelease->versionfilename;
         return;
     };
     return $job->fail("Couldn't create folder $path in DB") unless $folder && $folder->id;
@@ -164,6 +173,7 @@ sub _sync {
             }
             return;
         }
+        $obsrelease->next_file($file, $mtime) if $obsrelease;
         $cnt = $cnt + 1;
         $schema->resultset('File')->create({folder_id => $folder->id, name => $file, size => $size, mtime => $mtime, target => $target});
         return undef;
@@ -205,6 +215,7 @@ sub _sync {
     } else {
         $otherFolder->update({sync_last => \"CURRENT_TIMESTAMP(3)", sync_scheduled => \'coalesce(sync_scheduled, CURRENT_TIMESTAMP(3))'}) if $otherFolder;
     }
+    $schema->resultset('Folder')->add_rollout($proj->{id}, $obsrelease->versionmtime, $obsrelease->version, $obsrelease->versionfilename) if $obsrelease && $obsrelease->versionfilename;
 }
 
 1;

lib/MirrorCache/Task/MirrorScan.pm

@@ -111,6 +111,9 @@ sub _doscan {
     my $folder_on_mirrors = $schema->resultset('Server')->folder($folder_id);
     my $count = 0;
     my $perfect_count = 0;
+    my $proj = $schema->resultset('ProjectRollout')->rollout_file_for_folder($path);
+    my $proj_rollout_filename = '';
+    $proj_rollout_filename = $proj->{filename} if $proj;
     for my $folder_on_mirror (@$folder_on_mirrors) {
         my $server_id = $folder_on_mirror->{server_id};
         my $url = $folder_on_mirror->{url} . '/';
@@ -152,6 +155,14 @@ unless ($hasall) {
                     $href = uri_unescape($href);
                     1;
                 } or $href = '';
+                if ($proj_rollout_filename eq $href) {
+                    my $rollout_res = 0;
+                    my $rollout_err;
+                    eval {
+                        $rollout_res = $schema->resultset('ProjectRollout')->add_rollout_server($server_id, $proj->{id}, $proj->{epc});
+                    } or $rollout_err = $@ // 'no error';
+                    $job->note(rollout_res => $rollout_res, rollout_err => $rollout_err, at => datetime_now()) if $rollout_err;
+                }
             };
             my $end = sub {
                 $href = '';

lib/MirrorCache/WebAPI/Plugin/Dir.pm

@@ -207,7 +207,8 @@ sub _render_dir {
     $c->stat->redirect_to_root($dm, 0) unless $folder_id && $dm->folder_sync_last;
     return _render_dir_from_db($dm, $folder_id, $dir) if $folder_id && $dm->folder_sync_last;
 
-    my $pos = $rsFolder->get_sync_queue_position($dir);
+    my $pos = -1;
+    eval { $pos = $rsFolder->get_sync_queue_position($dir)->[0]; };
     return $c->render(status => 425, text => "Waiting in queue, at " . strftime("%Y-%m-%d %H:%M:%S", gmtime time) . " position: $pos");
 }
 

lib/MirrorCache/resources/migrations/Pg.sql

@@ -383,4 +383,19 @@ create table server_note (
     msg       varchar(512),
     primary key(hostname, dt)
 );
-
+-- 32 up
+create table project_rollout (
+    project_id bigint NOT NULL references project on delete cascade,
+    epc int NOT NULL,
+    dt timestamp,
+    version varchar(32),
+    filename varchar(256),
+    primary key(project_id, epc)
+);
+create table project_rollout_server (
+    server_id  bigint NOT NULL references server  on delete cascade,
+    project_id bigint NOT NULL references project on delete cascade,
+    epc int NOT NULL,
+    dt timestamp,
+    primary key(server_id, project_id, epc)
+);

lib/MirrorCache/resources/migrations/mysql.sql

@@ -384,4 +384,31 @@ create table server_note (
     msg       varchar(512),
     primary key(hostname, dt)
 );
+-- 32 up
+create table project_rollout (
+    project_id int NOT NULL,
+    epc int NOT NULL,
+    dt timestamp,
+    version varchar(32),
+    filename varchar(256),
+    primary key(project_id, epc)
+);
+
+create index if not exists i_project_rollout_project_id on project_rollout(project_id);
+
+alter table project_rollout add constraint `fk_project_rollout_project` FOREIGN KEY(project_id) references project(id) on delete cascade;
+
+create table project_rollout_server (
+    server_id  int NOT NULL,
+    project_id int NOT NULL,
+    epc int NOT NULL,
+    dt timestamp,
+    primary key(server_id, project_id, epc)
+);
+
+create index if not exists i_project_rollout_server_server_id on project_rollout_server(server_id);
+create index if not exists i_project_rollout_server_project_id on project_rollout_server(project_id);
+alter table project_rollout_server
+   add constraint `fk_project_rollout_server_server`  FOREIGN KEY(server_id)  references server(id)  on delete cascade,
+   add constraint `fk_project_rollout_server_project` FOREIGN KEY(project_id) references project(id) on delete cascade;
 

t/environ/14-project-rollout-iso.sh

@@ -0,0 +1,54 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+$mc/start
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+ap6=$(environ ap6)
+ap5=$(environ ap5)
+ap4=$(environ ap4)
+
+for x in $mc $ap7 $ap8 $ap6 $ap5 $ap4; do
+    mkdir -p $x/dt/{folder1,folder2,folder3}
+    mkdir -p $x/dt/project1/iso
+    mkdir -p $x/dt/project2/iso
+    echo $x/dt/project1/iso/proj1-Build1.1-Media.iso{,sha256} | xargs -n 1 touch
+    echo $x/dt/project2/iso/proj1-Snapshot240131-Media.iso{,.sha256} | xargs -n 1 touch
+done
+
+$ap4/start
+$ap5/start
+$ap6/start
+$ap7/start
+$ap8/start
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap6/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap5/print_address)','','t','cn','as'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap4/print_address)','','t','jp','as'"
+
+$mc/sql "insert into project(name,path,etalon) select 'proj1 ISO','/project1', 3"
+$mc/sql "insert into project(name,path,etalon) select 'proj 2 ISO','/project2', 3"
+
+$mc/backstage/job -e folder_sync -a '["/project1/iso"]'
+$mc/backstage/job -e mirror_scan -a '["/project1/iso"]'
+$mc/backstage/shoot
+
+$mc/sql_test 1 == 'select count(*) from project_rollout'
+$mc/sql_test 1.1 == 'select version from project_rollout'
+$mc/sql_test proj1-Build1.1-Media.iso == 'select filename from project_rollout'
+
+$mc/sql_test 5 == 'select count(*) from project_rollout_server'
+
+$mc/backstage/job -e folder_sync -a '["/project2/iso"]'
+$mc/backstage/job -e mirror_scan -a '["/project2/iso"]'
+$mc/backstage/shoot
+
+$mc/sql_test 2 == 'select count(*) from project_rollout'
+$mc/sql_test 240131 == 'select version from project_rollout where project_id = 2'
+
+echo success