openSUSE · andrii-suse · Oct 19, 2023 · Oct 16, 2023
diff --git a/lib/MirrorCache/Datamodule.pm b/lib/MirrorCache/Datamodule.pm
@@ -23,6 +23,8 @@ use Digest::SHA qw(sha1_hex);
 use Mojolicious::Types;
 use MirrorCache::Utils 'region_for_country';
 
+my $MCDEBUG = $ENV{MCDEBUG_DATAMODULE} // $ENV{MCDEBUG_ALL} // 0;
+
 has c => undef, weak => 1;
 
 my @ROUTES = ( '/browse', '/download' );
@@ -327,13 +329,18 @@ sub is_head($self) {
     return $self->_is_head;
 }
 
-sub redirect($self, $url) {
+sub redirect($self, $url, $skip_xtra = undef) {
     my $xtra = '';
-    if ($self->_original_path =~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) {
+
+    my $c = $self->c;
+    my $param = $c->req->params;
+    if (!$skip_xtra && $self->_original_path =~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) {
         $xtra = $1;
+        $xtra = substr($xtra, 1);
+        $param->append($xtra => 1);
     }
-
-    return $self->c->redirect_to($url . $xtra . $self->query1);
+    return $c->redirect_to($url) unless $param->to_hash;
+    return $c->redirect_to($url . '?' . $param->to_string);
 }
 
 sub accept($self) {
@@ -344,6 +351,7 @@ sub _init_headers($self) {
     $self->_agent('');
     $self->_browser('');
     my $headers = $self->c->req->headers;
+    $self->c->log->error($self->c->dumper("DATAMODULE HEADERS", $headers)) if $MCDEBUG;
     return unless $headers;
     if (my $agent = $headers->user_agent) {
         $self->_agent($agent);
@@ -378,17 +386,18 @@ sub _init_headers($self) {
     $self->_country($country) if $country;
     $self->_region($region)   if $region;
 
+    $self->c->log->error($self->c->dumper("DATAMODULE HEADERS ACCEPT", $headers->accept)) if $MCDEBUG;
     return unless $headers->accept;
 
-    $self->metalink(1)   if $headers->accept =~ m/\bapplication\/metalink/;
-    $self->meta4(1)      if $headers->accept =~ m/\bapplication\/metalink4/;
-    $self->zsync(1)      if $headers->accept =~ m/\bapplication\/x-zsync/;
+    $self->metalink(1)   if $headers->accept =~ m/\bapplication\/metalink/i;
+    $self->meta4(1)      if $headers->accept =~ m/\bapplication\/metalink4/i;
+    $self->zsync(1)      if $headers->accept =~ m/\bapplication\/x-zsync/i;
 
-    $self->accept_metalink(1)   if $headers->accept =~ m/\bapplication\/metalink/;
-    $self->accept_meta4(1)      if $headers->accept =~ m/\bapplication\/metalink4/;
-    $self->accept_zsync(1)      if $headers->accept =~ m/\bapplication\/x-zsync/;
+    $self->accept_metalink(1)   if $headers->accept =~ m/\bapplication\/metalink/i;
+    $self->accept_meta4(1)      if $headers->accept =~ m/\bapplication\/metalink4/i;
+    $self->accept_zsync(1)      if $headers->accept =~ m/\bapplication\/x-zsync/i;
 
-    $self->accept_all(1) if $headers->accept =~ m/\*\/\*/ && ($self->_original_path !~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/);
+    $self->accept_all(1) if $headers->accept =~ m/\*\/\*/;
 }
 
 sub _init_req($self) {
@@ -492,6 +501,8 @@ sub _init_path($self) {
         $self->_query($query);
         $self->_query1('?' . $query_string);
         $self->mirrorlist(1) if defined $query->param('mirrorlist');
+        $self->meta4(1)      if defined $query->param('meta4');
+        $self->metalink(1)   if defined $query->param('metalink');
         $self->zsync(1)      if defined $query->param('zsync');
         $self->torrent(1)    if defined $query->param('torrent');
         $self->magnet(1)     if defined $query->param('magnet');

diff --git a/lib/MirrorCache/WebAPI/Plugin/Dir.pm b/lib/MirrorCache/WebAPI/Plugin/Dir.pm
@@ -227,13 +227,18 @@ sub _redirect_project_ln_geo {
         }
 
         $c->log->error('pedantic: ' . ($dm->pedantic // 'undef')) if $MCDEBUG;
-        if ($path =~ m/(GNOME_.*|.*(Media|Current|Next))\.iso(\.sha256(\.asc)?)?/ && $dm->pedantic) {
-            my $ln = $root->detect_ln_in_the_same_folder($path);
+        if ($path =~ m/(GNOME_.*|.*(Media|[C|c]urrent|Next))\.iso(\.sha256(\.asc)?)?/ && $dm->pedantic) {
+            my $ln = $root->detect_ln_in_the_same_folder($dm->original_path);
+            my $extra = 1;
+            unless ($ln) {
+                $ln = $root->detect_ln_in_the_same_folder($path);
+                $extra = 0;
+            }
             $c->log->error("ln for $path : " . ($ln // 'null')) if $MCDEBUG;
             if ($ln) {
                 # redirect to the symlink
-                $c->log->error('redirect detected: ' . $ln) if $MCDEBUG;
-                $dm->redirect($dm->route . $ln);
+                $c->log->error('redirect detected: ' . $ln . ": " . $c->dumper($dm->accept_all, $dm->accept)) if $MCDEBUG;
+                $dm->redirect($dm->route . $ln, $extra && ($dm->accept_all || !$dm->accept));
                 return 1;
             }
         }
@@ -321,23 +326,49 @@ sub _render_stats_not_scanned {
 sub _local_render {
     my $dm     = shift;
     my $accept = shift;
-    return undef if $dm->extra && (!$accept || !$dm->accept_all);
+    my $c      = $dm->c;
+    $c->log->error($c->dumper('local_render : ', $dm->extra, $accept, $dm->accept_all)) if $MCDEBUG;
+    return undef if $dm->extra && (!$accept || $dm->accept);
+    $c->log->error($c->dumper('local_render2: ')) if $MCDEBUG;
     my ($path, $trailing_slash) = $dm->path;
     # we can just render top folders
     return _render_top_folders($dm) if @top_folders && $path eq '/';
-    return $root->render_file_if_nfs($dm, $path) if $root->is_remote;
+    my $original_path = $dm->original_path;
+
+    return $root->render_file_if_nfs($dm, $path) if $root->is_remote && ($original_path eq $path || (!$dm->extra && !$dm->accept));
+    return undef if $root->is_remote;
+    $c->log->error($c->dumper('local_render3: ')) if $MCDEBUG;
 
     # root is only local now
-    if (defined($dm->c->param('realpath'))) {
+    if (defined($c->param('realpath'))) {
         my $realpath = $root->realpath($path);
         return $dm->redirect($dm->route . $realpath . '/') if $realpath;
     }
     if ($root->is_dir($path)) {
         return $dm->redirect($dm->route . $path . '/') if !$trailing_slash && $path ne '/';
         return _render_dir($dm, $path);
     }
-    $dm->c->mirrorcache->render_file($path, $dm) if !$trailing_slash && $root->is_file($path);
-    return 1;
+    if (!$trailing_slash) {
+        if ($original_path ne $path && $root->is_file($original_path) && !$dm->accept) {
+            $c->log->error($c->dumper('local_render4 : ', $dm->extra)) if $MCDEBUG;
+            if ($accept) {
+                $root->render_file($dm, $original_path);
+            } else {
+                $c->mirrorcache->render_file($original_path, $dm);
+            }
+            return 1;
+        } elsif ($root->is_file($path) && !$dm->extra) {
+            $c->log->error($c->dumper('local_render5 : ', $dm->extra)) if $MCDEBUG;
+            if ($accept) {
+                $root->render_file($dm, $path);
+            } else {
+                $c->mirrorcache->render_file($path, $dm);
+            }
+            return 1;
+        }
+    }
+    $c->log->error($c->dumper('local_render6: ', $c->res->code)) if $MCDEBUG;
+    return $c->res->code;
 }
 
 sub _render_from_db {
@@ -359,7 +390,7 @@ sub _render_from_db {
         $dirname = $dm->root_subtree . ($folder_or_pattern? $path : $f->dirname) unless $dirname;
         $c->log->error($c->dumper('dirname:', $dirname, 'path:', $path, 'trail:', $trailing_slash)) if $MCDEBUG;
         if (my $folder = $rsFolder->find_folder_or_redirect($dirname)) {
-            $c->log->error("found redirect : $dirname -> ", $folder->{pathto}) if $MCDEBUG && $folder->{pathto};
+            $c->log->error($c->dumper("found redirect : $dirname -> ", $folder->{pathto})) if $MCDEBUG && $folder->{pathto};
             # return $dm->redirect($folder->{pathto} . $trailing_slash) if $folder->{pathto};
             my $folder_path = $folder->{pathto} ? $folder->{pathto} : $folder->{path};
 	    return $c->render(status => 404, text => "path {$path} not found!!") unless $folder_path;
@@ -369,7 +400,7 @@ sub _render_from_db {
             } else {
                 $realpath_subtree = $root->realpath($dm->root_subtree . ($folder_or_pattern? $path : $f->dirname)) // $dirname;
             }
-            $c->log->error('RENDER - REALPATH_SUBTREE : ', $realpath_subtree) if $MCDEBUG;
+            $c->log->error('RENDER - REALPATH_SUBTREE : ' . $realpath_subtree) if $MCDEBUG;
             if ($dirname eq $realpath_subtree) {
                 if ($dirname eq $f->dirname || $folder_or_pattern) {
                     $dm->folder_id($folder->{id});
@@ -404,14 +435,14 @@ sub _render_from_db {
                 my $filename = $file->{name} if $file;
                 if ($dm->zsync && !$dm->accept_zsync && $file && $filename && '.zsync' eq substr $filename, -length('.zsync')) {
                     $dm->zsync(0);
-                    $dm->accept_all(1);
+                    # $dm->accept_all(1);
                     $dm->_path($dm->path . '.zsync');
                     $path = $path . '.zsync';
                 }
 
                 if ($file->{target}) {
                     # redirect to the symlink
-                    $dm->redirect($dm->route . $dirname . '/' . $file->{target});
+                    $dm->redirect($dm->route . $dirname . '/' . $file->{target}, ($dm->accept_all || !$dm->accept));
                 } else {
                     $dm->file_id($file->{id});
                     # find a mirror for it
@@ -429,16 +460,21 @@ sub _guess_what_to_render {
     my $c    = $dm->c;
     my $tx   = $c->render_later->tx;
     my ($path, $trailing_slash) = $dm->path;
+    $c->log->error('guess what to render: ' . $path) if $MCDEBUG;
 
     if ($dm->extra) {
-        $c->log->error('guess what to render extra : ', $dm->extra) if $MCDEBUG;
-        return $root->render_file($dm, $path) if $dm->accept_all && !$trailing_slash;
+        $c->log->error($c->dumper('guess what to render extra : ', $dm->extra, $dm->accept_all)) if $MCDEBUG;
+        return $root->render_file($dm, $dm->original_path) if $dm->accept_all && !$trailing_slash && $dm->accept;
+        if (!$root->is_remote && !$dm->accept) { # for local we can check if it is the file we requested
+            return $root->render_file($dm, $dm->original_path) if $root->is_file($dm->original_path);
+        }
         # the file is unknown, we cannot show generate meither mirrorlist or metalink
         my $res = $c->render(status => 425, text => "The file is unknown, retry later");
         # log miss here even thoough we haven't rendered anything
         $c->stat->redirect_to_root($dm, 0);
         return $res;
     }
+    return $c->render(status => 404, text => "Not found") unless $root->is_remote;
 
     my $rootlocation = $root->location;
     my $url  = $rootlocation . $path;
@@ -656,19 +692,26 @@ sub _render_small {
     my $dm = shift;
     my $root_nfs = $mc_config->root_nfs;
     my $small_file_size = $mc_config->small_file_size;
-    return undef unless $small_file_size && ($root_nfs || !$root->is_remote );
+    my $c=$dm->c;
+    $c->log->error('DIR::render_small1') if $MCDEBUG;
+    return undef unless ($small_file_size && ($root_nfs || !$root->is_remote));
     $dm->_init_path;
-    return undef if ($dm->metalink && !$dm->accept_all) || ($dm->meta4 && !$dm->accept_all) || $dm->mirrorlist || $dm->zsync;
+    $c->log->error('DIR::render_small2') if $MCDEBUG;
+    return undef if ($dm->metalink && $dm->accept) || ($dm->meta4 && $dm->accept) || $dm->mirrorlist || $dm->zsync;
+    $c->log->error('DIR::render_small3') if $MCDEBUG;
     my ($path, undef) = $dm->path;
     my $full;
     return $root->render_file_if_small($dm, $path, $small_file_size) unless $root_nfs;
+    $c->log->error('DIR::render_small4') if $MCDEBUG;
+    my $original_path = $dm->path;
+    return undef if $original_path ne $path || $dm->extra;
+    $c->log->error($c->dumper('DIR::render_small5', $original_path, $path, $dm->extra)) if $MCDEBUG;
     $full = $root_nfs . $path;
     my $size;
     eval { $size = -s $full if -f $full; };
     return undef unless (defined $size) && $size <= $small_file_size;
-    my $c = $dm->c;
-    $c->render_file(filepath => $full, content_type => $dm->mime);
-    return 1;
+    $c->log->error('DIR::render_small6') if $MCDEBUG;
+    return $root->render_file($dm, $path, 1, 1);
 }
 
 # if we don't render file directly - we set max-age to short value, because redirect or metalink may change

diff --git a/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm b/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
@@ -150,7 +150,7 @@ sub register {
         }
 
         if (!$file) {
-            return $c->render(status => 404, text => "File not found");
+            return undef;
         }
         $c->log->error($c->dumper('RENDER FILE_ID', $file->{id})) if $MCDEBUG;
         $c->res->headers->vary('Accept, COUNTRY');

diff --git a/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm b/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
@@ -141,10 +141,10 @@ sub is_dir {
 }
 
 sub render_file {
-    my ($self, $dm, $filepath, $not_miss) = @_;
+    my ($self, $dm, $filepath, $not_miss, $from_nfs) = @_;
     my $c = $dm->c;
     my $nfs = $self->rootnfs;
-    if ($nfs && $dm->must_render_from_root && -f $nfs . $filepath) {
+    if ($nfs && ($dm->must_render_from_root || $from_nfs) && -f ($nfs . $filepath)) {
         $c->render_file(filepath => $nfs . $filepath, content_type => $dm->mime, content_disposition => 'inline');
         $c->stat->redirect_to_root($dm, $not_miss);
         return 1;

diff --git a/t/environ/02-files-hashes.sh b/t/environ/02-files-hashes.sh
@@ -17,8 +17,18 @@ for x in $mc; do
     echo 1111111111 > $x/dt/folder1/file1.1.dat
     echo 1111111111 > $x/dt/folder1/file2.1.dat
     echo 2345 > $x/dt/folder1/file2.1.dat.zsync
+    echo 2345 > $x/dt/folder1/fileX.dat.zsync
 done
 
+$mc/curl -I /download/folder1/file2.1.dat.zsync | grep '200 OK'
+$mc/curl -I /download/folder1/fileX.dat.zsync   | grep '200 OK'
+$mc/curl -I /download/folder1/file2.1.dat.meta4 | grep '425'
+$mc/curl -H 'Accept: Application/x-zsync' -I /download/folder1/file2.1.dat | grep '425'
+$mc/curl -H 'Accept: Application/metalink+xml' -I /download/folder1/file2.1.dat | grep '425'
+$mc/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file2.1.dat | grep '200 OK'
+$mc/curl -H 'Accept: Application/metalink+xml, Application/x-zsync' -I /download/folder1/file2.1.dat | grep '425'
+$mc/curl -H 'Accept: Application/metalink+xml, Application/x-zsync, */*' -I /download/folder1/file2.1.dat | grep '200 OK'
+
 # force scan
 $mc/backstage/job -e folder_sync -a '["/folder1"]'
 $mc/backstage/shoot

diff --git a/t/environ/03-headquarter-subsidiaries-hashes.sh b/t/environ/03-headquarter-subsidiaries-hashes.sh
@@ -80,8 +80,12 @@ for i in 9 6 7 8; do
     mc$i/backstage/shoot
     mc$i/backstage/shoot -q hashes
     mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id = file_id where name='file-Media.iso'"
-    for x in '' .metalink .mirrorlist; do
-        mc$i/curl -I /folder1/file-Media.iso$x | grep -C 10 302 | grep /folder1/file4.1.dat$x | grep -v /download/folder1/file4.1.dat$x
+    for x in '' metalink mirrorlist; do
+        ext=""
+        par=""
+        [ -z "$x"] || ext=.$x
+        [ -z "$x"] || par=?$x
+        mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$par | grep -v /download/folder1/file4.1.dat
     done
 done
 

diff --git a/t/environ/04-remote-current.sh b/t/environ/04-remote-current.sh
@@ -18,8 +18,10 @@ for x in $ap7 $ap8 $ap9; do
     echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs -n 1 touch
     sha256sum $x/dt/folder1/file1.1-Media.iso > $x/dt/folder1/file1.1-Media.iso.sha256
     echo 111112 > $x/dt/folder1/file2.1-Media.iso
+    echo 111113 > $x/dt/folder1/file2.1-Media.iso.zsync
     sha256sum $x/dt/folder1/file2.1-Media.iso > $x/dt/folder1/file2.1-Media.iso.sha256
     ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s file1.1-Media.iso.sha256 file-Media.iso.sha256 )
+    ( cd $x/dt/folder1 && ln -s file2.1-Media.iso.zsync file-Media.iso.zsync )
 done
 
 for x in $ap7 $ap8 $ap9; do
@@ -45,6 +47,9 @@ $mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /dow
 $mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256
 $mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  "
 
+$mc/curl -I /download/folder1/file2.1-Media.iso.zsync | grep --color=never -P 'Location: http://127.0.0.1:1324/folder1/file2.1-Media.iso.zsync\r$'
+$mc/curl -I /download/folder1/file-Media.iso.zsync    | grep --color=never -P 'file2.1-Media.iso.zsync\r$'
+
 echo now change the symlink and make sure redirect changes
 (
     cd $ap9/dt/folder1

diff --git a/t/environ/04-remote-nfs.sh b/t/environ/04-remote-nfs.sh
@@ -26,6 +26,9 @@ ap7=$(environ ap7)
 for x in $ap7 $ap8 $ap9 $mc1 $mc2; do
     mkdir -p $x/dt/{folder1,folder2,folder3}
     echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch
+    echo 111112 > $x/dt/folder1/file2.1-Media.iso
+    echo 111113 > $x/dt/folder1/file2.1-Media.iso.zsync
+    ( cd $x/dt/folder1 && ln -s file2.1-Media.iso.zsync file-Media.iso.zsync )
 
     mkdir -p $x/dt/updates/tool
     (
@@ -77,4 +80,13 @@ $mc2/backstage/shoot
 echo now we learned about all 3 folders
 $mc2/sql_test 3 == 'select count(*) from folder'
 
+$mc2/curl -H 'Accept: Application/x-zsync' -IL /download/folder1/file-Media.iso.zsync    | grep '404 Not Found'
+$mc2/curl -H 'Accept: Application/x-zsync' -I  /download/folder1/file2.1-Media.iso.zsync | grep '404 Not Found' # we don't have zhashes for this file
+$mc2/curl -H 'Accept: Application/x-zsync' -I  /download/folder1/file2.1-Media.iso | grep '404 Not Found' # we don't have zhashes for this file
+$mc2/curl -H 'Accept: Application/metalink+xml' -I /download/folder1/file2.1-Media.iso | grep '200 OK'
+$mc2/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file2.1-Media.iso | grep '200 OK'
+$mc2/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file-Media.iso.zsync | grep --color=never -P '/download/folder1/file2.1-Media.iso.zsync\r$'
+
+$mc2/curl -I /download/folder1/file-Media.iso.zsync | grep --color=never -P '/download/folder1/file2.1-Media.iso.zsync\r$'
+
 echo success