Redirect Current.iso when no nfs mount

Previosly MirrorCache could properly detect correct redirect for Current.iso files only when files were available on disk.
openSUSE · Jan 10, 2023 · a462426 · a462426
1 parent 4909941
commit a462426
Show file tree

Hide file tree

Showing 6 changed files with 169 additions and 61 deletions.
diff --git a/lib/MirrorCache/WebAPI/Plugin/Dir.pm b/lib/MirrorCache/WebAPI/Plugin/Dir.pm
@@ -176,15 +176,18 @@ sub _redirect_project_ln_geo {
             $c->stat->redirect_to_region($dm);
             return 1;
         }
-    }
 
-    my $ln = $root->detect_ln($path);
-    if ($ln) {
-        # redirect to the symlink
-        $c->log->error('redirect detected: ' . $ln) if $MCDEBUG;
-        $dm->redirect($dm->route . $ln);
-        $c->stat->redirect_to_region($dm);
-        return 1;
+        $c->log->error('pedantic: ' . $dm->pedantic) if $MCDEBUG;
+        if ($path =~ m/.*(Media|Current)\.iso(\.sha256(\.asc)?)?/ && $dm->pedantic) {
+            my $ln = $root->detect_ln_in_the_same_folder($path);
+            $c->log->error("ln for $path : " . ($ln // 'null')) if $MCDEBUG;
+            if ($ln) {
+                # redirect to the symlink
+                $c->log->error('redirect detected: ' . $ln) if $MCDEBUG;
+                $dm->redirect($dm->route . $ln);
+                return 1;
+            }
+        }
     }
     return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
     return undef if $dm->must_render_from_root;
@@ -314,7 +317,7 @@ sub _render_from_db {
             my $xtra = '';
             $xtra = '.zsync' if $dm->zsync && !$dm->accept_zsync;
             my $file;
-            $c->log->error($c->dumper('parent_folder:', $parent_folder)) if $MCDEBUG;
+            $c->log->error($c->dumper('parent_folder:', $parent_folder->path)) if $MCDEBUG;
             $file = $schema->resultset('File')->find_with_hash($parent_folder->id, $f->basename, $xtra, $dm->regex, $dm->glob_regex) if $parent_folder;
             $c->log->error($c->dumper('file:', $f->basename, $file)) if $MCDEBUG;
 
@@ -427,8 +430,6 @@ sub _guess_what_to_render {
 }
 
 sub _by_filename {
-   delete $b->{desc} unless $b->{desc};
-   delete $a->{desc} unless $a->{desc};
    versioncmp(lc($b->{dir} // ''),  lc($a->{dir} // '')) ||
    versioncmp(lc($a->{name} // ''), lc($b->{name} // ''));
 }
@@ -497,31 +498,31 @@ sub _render_dir_from_db {
         my $basename = $child->{name};
         my $size     = $child->{size};
         my $mtime    = $child->{mtime};
+        my $desc     = $folderDesc{$c->mcbranding}{$dir}{$basename};
         if ($json) {
             push @files, {
                 name  => $basename,
                 size  => $size,
                 mtime => $mtime,
-                desc  => $folderDesc{$c->mcbranding}{$dir}{$basename},
             };
-            next;
-        }
-        $size        = MirrorCache::Utils::human_readable_size($size) if $size;
-        $mtime       = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
+        } else {
+            $size        = MirrorCache::Utils::human_readable_size($size) if $size;
+            $mtime       = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
 
-        my $is_dir    = '/' eq substr($basename, -1)? 1 : 0;
-        my $encoded   = Encode::decode_utf8( './' . $basename );
-        my $mime_type = $dm->mime || 'text/plain';
+            my $is_dir    = '/' eq substr($basename, -1)? 1 : 0;
+            my $encoded   = Encode::decode_utf8( './' . $basename );
+            my $mime_type = $dm->mime || 'text/plain';
 
-        push @files, {
-            url   => $encoded,
-            name  => $basename,
-            size  => $size,
-            type  => $mime_type,
-            mtime => $mtime,
-            dir   => $is_dir,
-            desc  => $folderDesc{$c->mcbranding}{$dir}{$basename},
-        };
+            push @files, {
+                url   => $encoded,
+                name  => $basename,
+                size  => $size,
+                type  => $mime_type,
+                mtime => $mtime,
+                dir   => $is_dir,
+            };
+        }
+        $files[-1]->{desc} = $desc if $desc;
     }
     my @items = sort _by_filename @files;
     return $c->render( json => { data => \@items } ) if $dm->jsontable;
@@ -549,32 +550,31 @@ sub _render_dir_local {
         $basename = $basename . '/' if $stat && -d $stat;
         my $size     = $stat->size if $stat;
         my $mtime    = $stat->mtime if $stat;
+        my $desc     = $folderDesc{$c->mcbranding}{$dir}{$basename};
         if ($json) {
             push @files, {
                 name  => $basename,
                 size  => $size,
                 mtime => $mtime,
-                desc  => $folderDesc{$c->mcbranding}{$dir}{$basename},
             };
-            next;
-        }
-
-        $size        = MirrorCache::Utils::human_readable_size($size) if $size;
-        $mtime       = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
+        } else {
+            $size        = MirrorCache::Utils::human_readable_size($size) if $size;
+            $mtime       = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
 
-        my $is_dir    = '/' eq substr($basename, -1)? 1 : 0;
-        my $encoded   = Encode::decode_utf8( './' . $basename );
-        my $mime_type = $dm->mime || 'text/plain';
+            my $is_dir    = '/' eq substr($basename, -1)? 1 : 0;
+            my $encoded   = Encode::decode_utf8( './' . $basename );
+            my $mime_type = $dm->mime || 'text/plain';
 
-        push @files, {
-            url   => $encoded,
-            name  => $basename,
-            size  => $size,
-            type  => $mime_type,
-            mtime => $mtime,
-            dir   => $is_dir,
-            desc  => $folderDesc{$c->mcbranding}{$dir}{$basename},
-        };
+            push @files, {
+                url   => $encoded,
+                name  => $basename,
+                size  => $size,
+                type  => $mime_type,
+                mtime => $mtime,
+                dir   => $is_dir,
+            };
+        }
+        $files[-1]->{desc} = $desc if $desc;
     }
     my @items = sort _by_filename @files;
     return $c->render( json => { data => \@items } ) if $dm->jsontable;

diff --git a/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm b/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
@@ -149,7 +149,7 @@ if ($TOP_FOLDERS) {
     }
 }
 
-sub _detect_ln {
+sub _detect_ln_in_the_same_folder {
     my ($dir, $file) = @_;
     return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
 
@@ -167,10 +167,10 @@ sub _detect_ln {
     return undef;
 }
 
-sub detect_ln {
+sub detect_ln_in_the_same_folder {
     my ($self, $path) = @_;
     my $f = Mojo::File->new($path);
-    my $res = _detect_ln($f->dirname, $f->basename);
+    my $res = _detect_ln_in_the_same_folder($f->dirname, $f->basename);
     return undef unless $res;
     return $f->dirname . '/' . $res;
 }

diff --git a/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm b/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
@@ -171,10 +171,28 @@ sub looks_like_file {
     return 1;
 };
 
-sub _detect_ln {
-    return undef unless $nfs;
-    my ($dir, $file) = @_;
-    return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
+sub _detect_ln_in_the_same_folder {
+    my ($self, $dir, $file) = @_;
+
+    unless ($nfs) {
+        my $dir1 = $dir . '/';
+        my $rootlocation = $self->rooturl;
+        my $url = $rootlocation . $dir1 . $file;
+        my $ua = Mojo::UserAgent->new->max_redirects(0)->request_timeout(2);
+        my $tx = $ua->head($url, {'User-Agent' => 'MirrorCache/detect_redirect'});
+        my $res = $tx->res;
+
+        # redirect on oneself
+        if ($res->is_redirect && $res->headers) {
+            my $location = $res->headers->location;
+            my $url1 = $rootlocation . $dir1;
+            if ($location && $url1 eq substr($location, 0, length($url1))) {
+                my $ln = substr($location, length($url1));
+                return $ln if -1 == index($ln, '/');
+            }
+        }
+        return undef;
+    }
 
     my $dest;
     eval {
@@ -191,11 +209,11 @@ sub _detect_ln {
     return $res;
 }
 
-sub detect_ln {
-    return undef unless $nfs;
+# this is simillar to self->realpath, just detects symlinks in current folder
+sub detect_ln_in_the_same_folder {
     my ($self, $path) = @_;
     my $f = Mojo::File->new($path);
-    my $res = _detect_ln($f->dirname, $f->basename);
+    my $res = $self->_detect_ln_in_the_same_folder($f->dirname, $f->basename);
     return undef unless $res;
     return $f->dirname . '/' . $res;
 }
@@ -258,7 +276,7 @@ sub _foreach_filename_html {
 
         if ($t && ($href20 eq substr($t,0,20))) {
             if ($desc{name} && (!$P || $desc{name} =~ $P)) {
-                my $target = _detect_ln($dir, $desc{name});
+                my $target = $self->_detect_ln_in_the_same_folder($dir, $desc{name});
                 $sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
                 %desc = ();
             }
@@ -299,7 +317,7 @@ sub _foreach_filename_html {
         $p->parse($chunk);
     }
     if ($desc{name} && (!$P || $desc{name} =~ $P)) {
-        my $target = detect_ln($dir, $desc{name});
+        my $target = $self->_detect_ln_in_the_same_folder($dir, $desc{name});
         $sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
         %desc = ();
     }

diff --git a/t/environ/04-remote-current-no-nfs.sh b/t/environ/04-remote-current-no-nfs.sh
@@ -0,0 +1,69 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+ap9=$(environ ap9)
+
+$mc/gen_env \
+    MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
+    MIRRORCACHE_ROOT=http://$($ap9/print_address) \
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+
+for x in $ap7 $ap8 $ap9; do
+    mkdir -p $x/dt/{folder1,folder2,folder3}
+    echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs -n 1 touch
+    sha256sum $x/dt/folder1/file1.1-Media.iso > $x/dt/folder1/file1.1-Media.iso.sha256
+    echo 111112 > $x/dt/folder1/file2.1-Media.iso
+    sha256sum $x/dt/folder1/file2.1-Media.iso > $x/dt/folder1/file2.1-Media.iso.sha256
+    ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s file1.1-Media.iso.sha256 file-Media.iso.sha256 )
+done
+
+echo '    RewriteEngine On
+    RewriteBase "/"
+    RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file1.1-Media.iso$1 [R]
+' > $ap9/directory-rewrite.conf
+
+echo 'LoadModule rewrite_module    /usr/lib64/apache2-prefork/mod_rewrite.so' > $ap9/extra-rewrite.conf
+
+for x in $ap7 $ap8 $ap9; do
+    $x/start
+done
+
+$mc/start
+$mc/status
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','us','na'"
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+
+
+$mc/sql "select * from file"
+
+################################################
+# Test unversioned Media.iso is redirected to file which is metioned inside corresponding Media.iso.sha256
+$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  "
+
+echo now change the symlink and make sure redirect changes
+echo '    RewriteEngine On
+    RewriteBase "/"
+    RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file2.1-Media.iso$1 [R]
+' > $ap9/directory-rewrite.conf
+
+$ap9/stop
+$ap9/start
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e  "
+
+echo success
+
diff --git a/t/environ/04-remote-mc.sh b/t/environ/04-remote-mc.sh
@@ -3,6 +3,7 @@ set -ex
 
 mc=$(environ mc $(pwd))
 
+# we name it the same way as in test 04-remote-nginx.sh to simplify diff
 ng9=$(environ mc2 $(pwd))
 
 MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
@@ -200,3 +201,23 @@ $mc/curl /download/folder1/ | grep -B2 ln-Media.iso | grep '10 Byte'
 
 $mc/curl -IL /download/folder1/$e | grep '200 OK'
 $mc/curl -I  /download/folder1/$e | grep -C20 '302 Found' | grep -E "$($ng7/print_address)|$($ng8/print_address)" | grep "/folder1/$e"
+
+
+for x in $mc $ng9; do
+  for pattern in 'P=*2.1*' 'GLOB=*2.1*' 'REGEX=.*2\.1.*'; do
+    for extra in '' '&json' '&jsontable'; do
+        echo $x $pattern $extra
+        $x/curl /download/folder1/?$pattern$extra | grep -o file2.1.dat
+        rc=0
+        $x/curl /download/folder1/?$pattern$extra | grep -o file1.1.dat || rc=$?
+        test $rc -gt 0
+    done
+  done
+done
+
+$mc/curl "/download/folder1/?GLOB=*2.1*&mirrorlist"     | grep '<li>Filename: file2.1.dat</li>'
+$mc/curl "/download/folder1/?REGEX=.*2\.1.*&mirrorlist" | grep '<li>Filename: file2.1.dat</li>'
+
+
+echo success
+
diff --git a/templates/dir.html.ep b/templates/dir.html.ep
@@ -5,7 +5,7 @@
 % my $full_path = $cur_path;
 % $full_path = $route . $cur_path unless $route eq '/';
 % my @breadcrumbs = split '/', $full_path;
-% my $bc_last = pop @breadcrumbs;
+% my $bc_last = pop @breadcrumbs // '';
 %= include 'layouts/info'
 
 % my $mc_branding = eval '$branding' // '';
@@ -21,11 +21,11 @@
 
 <style type='text/css'>
 
-a {
+tr.td.a {
   display: flex;
 }
 
-a::after{
+tr.td.a::after{
   content: var(--desc);
   color: grey;
   margin-left: auto;