diff --git a/lib/MirrorCache/Datamodule.pm b/lib/MirrorCache/Datamodule.pm index bd3b8e92..3001b91c 100644 --- a/lib/MirrorCache/Datamodule.pm +++ b/lib/MirrorCache/Datamodule.pm @@ -23,6 +23,8 @@ use Digest::SHA qw(sha1_hex); use Mojolicious::Types; use MirrorCache::Utils 'region_for_country'; +my $MCDEBUG = $ENV{MCDEBUG_DATAMODULE} // $ENV{MCDEBUG_ALL} // 0; + has c => undef, weak => 1; my @ROUTES = ( '/browse', '/download' ); @@ -327,13 +329,18 @@ sub is_head($self) { return $self->_is_head; } -sub redirect($self, $url) { +sub redirect($self, $url, $skip_xtra = undef) { my $xtra = ''; - if ($self->_original_path =~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) { + + my $c = $self->c; + my $param = $c->req->params; + if (!$skip_xtra && $self->_original_path =~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) { $xtra = $1; + $xtra = substr($xtra, 1); + $param->append($xtra => 1); } - - return $self->c->redirect_to($url . $xtra . $self->query1); + return $c->redirect_to($url) unless $param->to_hash; + return $c->redirect_to($url . '?' . $param->to_string); } sub accept($self) { @@ -344,6 +351,7 @@ sub _init_headers($self) { $self->_agent(''); $self->_browser(''); my $headers = $self->c->req->headers; + $self->c->log->error($self->c->dumper("DATAMODULE HEADERS", $headers)) if $MCDEBUG; return unless $headers; if (my $agent = $headers->user_agent) { $self->_agent($agent); @@ -378,17 +386,18 @@ sub _init_headers($self) { $self->_country($country) if $country; $self->_region($region) if $region; + $self->c->log->error($self->c->dumper("DATAMODULE HEADERS ACCEPT", $headers->accept)) if $MCDEBUG; return unless $headers->accept; - $self->metalink(1) if $headers->accept =~ m/\bapplication\/metalink/; - $self->meta4(1) if $headers->accept =~ m/\bapplication\/metalink4/; - $self->zsync(1) if $headers->accept =~ m/\bapplication\/x-zsync/; + $self->metalink(1) if $headers->accept =~ m/\bapplication\/metalink/i; + $self->meta4(1) if $headers->accept =~ m/\bapplication\/metalink4/i; + $self->zsync(1) if $headers->accept =~ m/\bapplication\/x-zsync/i; - $self->accept_metalink(1) if $headers->accept =~ m/\bapplication\/metalink/; - $self->accept_meta4(1) if $headers->accept =~ m/\bapplication\/metalink4/; - $self->accept_zsync(1) if $headers->accept =~ m/\bapplication\/x-zsync/; + $self->accept_metalink(1) if $headers->accept =~ m/\bapplication\/metalink/i; + $self->accept_meta4(1) if $headers->accept =~ m/\bapplication\/metalink4/i; + $self->accept_zsync(1) if $headers->accept =~ m/\bapplication\/x-zsync/i; - $self->accept_all(1) if $headers->accept =~ m/\*\/\*/ && ($self->_original_path !~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/); + $self->accept_all(1) if $headers->accept =~ m/\*\/\*/; } sub _init_req($self) { @@ -492,6 +501,8 @@ sub _init_path($self) { $self->_query($query); $self->_query1('?' . $query_string); $self->mirrorlist(1) if defined $query->param('mirrorlist'); + $self->meta4(1) if defined $query->param('meta4'); + $self->metalink(1) if defined $query->param('metalink'); $self->zsync(1) if defined $query->param('zsync'); $self->torrent(1) if defined $query->param('torrent'); $self->magnet(1) if defined $query->param('magnet'); diff --git a/lib/MirrorCache/WebAPI/Plugin/Dir.pm b/lib/MirrorCache/WebAPI/Plugin/Dir.pm index 2cb55c62..8927a281 100644 --- a/lib/MirrorCache/WebAPI/Plugin/Dir.pm +++ b/lib/MirrorCache/WebAPI/Plugin/Dir.pm @@ -227,13 +227,18 @@ sub _redirect_project_ln_geo { } $c->log->error('pedantic: ' . ($dm->pedantic // 'undef')) if $MCDEBUG; - if ($path =~ m/(GNOME_.*|.*(Media|Current|Next))\.iso(\.sha256(\.asc)?)?/ && $dm->pedantic) { - my $ln = $root->detect_ln_in_the_same_folder($path); + if ($path =~ m/(GNOME_.*|.*(Media|[C|c]urrent|Next))\.iso(\.sha256(\.asc)?)?/ && $dm->pedantic) { + my $ln = $root->detect_ln_in_the_same_folder($dm->original_path); + my $extra = 1; + unless ($ln) { + $ln = $root->detect_ln_in_the_same_folder($path); + $extra = 0; + } $c->log->error("ln for $path : " . ($ln // 'null')) if $MCDEBUG; if ($ln) { # redirect to the symlink - $c->log->error('redirect detected: ' . $ln) if $MCDEBUG; - $dm->redirect($dm->route . $ln); + $c->log->error('redirect detected: ' . $ln . ": " . $c->dumper($dm->accept_all, $dm->accept)) if $MCDEBUG; + $dm->redirect($dm->route . $ln, $extra && ($dm->accept_all || !$dm->accept)); return 1; } } @@ -321,14 +326,21 @@ sub _render_stats_not_scanned { sub _local_render { my $dm = shift; my $accept = shift; - return undef if $dm->extra && (!$accept || !$dm->accept_all); + my $c = $dm->c; + $c->log->error($c->dumper('local_render : ', $dm->extra, $accept, $dm->accept_all)) if $MCDEBUG; + return undef if $dm->extra && (!$accept || $dm->accept); + $c->log->error($c->dumper('local_render2: ')) if $MCDEBUG; my ($path, $trailing_slash) = $dm->path; # we can just render top folders return _render_top_folders($dm) if @top_folders && $path eq '/'; - return $root->render_file_if_nfs($dm, $path) if $root->is_remote; + my $original_path = $dm->original_path; + + return $root->render_file_if_nfs($dm, $path) if $root->is_remote && ($original_path eq $path || (!$dm->extra && !$dm->accept)); + return undef if $root->is_remote; + $c->log->error($c->dumper('local_render3: ')) if $MCDEBUG; # root is only local now - if (defined($dm->c->param('realpath'))) { + if (defined($c->param('realpath'))) { my $realpath = $root->realpath($path); return $dm->redirect($dm->route . $realpath . '/') if $realpath; } @@ -336,8 +348,27 @@ sub _local_render { return $dm->redirect($dm->route . $path . '/') if !$trailing_slash && $path ne '/'; return _render_dir($dm, $path); } - $dm->c->mirrorcache->render_file($path, $dm) if !$trailing_slash && $root->is_file($path); - return 1; + if (!$trailing_slash) { + if ($original_path ne $path && $root->is_file($original_path) && !$dm->accept) { + $c->log->error($c->dumper('local_render4 : ', $dm->extra)) if $MCDEBUG; + if ($accept) { + $root->render_file($dm, $original_path); + } else { + $c->mirrorcache->render_file($original_path, $dm); + } + return 1; + } elsif ($root->is_file($path) && !$dm->extra) { + $c->log->error($c->dumper('local_render5 : ', $dm->extra)) if $MCDEBUG; + if ($accept) { + $root->render_file($dm, $path); + } else { + $c->mirrorcache->render_file($path, $dm); + } + return 1; + } + } + $c->log->error($c->dumper('local_render6: ', $c->res->code)) if $MCDEBUG; + return $c->res->code; } sub _render_from_db { @@ -359,7 +390,7 @@ sub _render_from_db { $dirname = $dm->root_subtree . ($folder_or_pattern? $path : $f->dirname) unless $dirname; $c->log->error($c->dumper('dirname:', $dirname, 'path:', $path, 'trail:', $trailing_slash)) if $MCDEBUG; if (my $folder = $rsFolder->find_folder_or_redirect($dirname)) { - $c->log->error("found redirect : $dirname -> ", $folder->{pathto}) if $MCDEBUG && $folder->{pathto}; + $c->log->error($c->dumper("found redirect : $dirname -> ", $folder->{pathto})) if $MCDEBUG && $folder->{pathto}; # return $dm->redirect($folder->{pathto} . $trailing_slash) if $folder->{pathto}; my $folder_path = $folder->{pathto} ? $folder->{pathto} : $folder->{path}; return $c->render(status => 404, text => "path {$path} not found!!") unless $folder_path; @@ -369,7 +400,7 @@ sub _render_from_db { } else { $realpath_subtree = $root->realpath($dm->root_subtree . ($folder_or_pattern? $path : $f->dirname)) // $dirname; } - $c->log->error('RENDER - REALPATH_SUBTREE : ', $realpath_subtree) if $MCDEBUG; + $c->log->error('RENDER - REALPATH_SUBTREE : ' . $realpath_subtree) if $MCDEBUG; if ($dirname eq $realpath_subtree) { if ($dirname eq $f->dirname || $folder_or_pattern) { $dm->folder_id($folder->{id}); @@ -404,14 +435,14 @@ sub _render_from_db { my $filename = $file->{name} if $file; if ($dm->zsync && !$dm->accept_zsync && $file && $filename && '.zsync' eq substr $filename, -length('.zsync')) { $dm->zsync(0); - $dm->accept_all(1); + # $dm->accept_all(1); $dm->_path($dm->path . '.zsync'); $path = $path . '.zsync'; } if ($file->{target}) { # redirect to the symlink - $dm->redirect($dm->route . $dirname . '/' . $file->{target}); + $dm->redirect($dm->route . $dirname . '/' . $file->{target}, ($dm->accept_all || !$dm->accept)); } else { $dm->file_id($file->{id}); # find a mirror for it @@ -429,16 +460,21 @@ sub _guess_what_to_render { my $c = $dm->c; my $tx = $c->render_later->tx; my ($path, $trailing_slash) = $dm->path; + $c->log->error('guess what to render: ' . $path) if $MCDEBUG; if ($dm->extra) { - $c->log->error('guess what to render extra : ', $dm->extra) if $MCDEBUG; - return $root->render_file($dm, $path) if $dm->accept_all && !$trailing_slash; + $c->log->error($c->dumper('guess what to render extra : ', $dm->extra, $dm->accept_all)) if $MCDEBUG; + return $root->render_file($dm, $dm->original_path) if $dm->accept_all && !$trailing_slash && $dm->accept; + if (!$root->is_remote && !$dm->accept) { # for local we can check if it is the file we requested + return $root->render_file($dm, $dm->original_path) if $root->is_file($dm->original_path); + } # the file is unknown, we cannot show generate meither mirrorlist or metalink my $res = $c->render(status => 425, text => "The file is unknown, retry later"); # log miss here even thoough we haven't rendered anything $c->stat->redirect_to_root($dm, 0); return $res; } + return $c->render(status => 404, text => "Not found") unless $root->is_remote; my $rootlocation = $root->location; my $url = $rootlocation . $path; @@ -656,19 +692,26 @@ sub _render_small { my $dm = shift; my $root_nfs = $mc_config->root_nfs; my $small_file_size = $mc_config->small_file_size; - return undef unless $small_file_size && ($root_nfs || !$root->is_remote ); + my $c=$dm->c; + $c->log->error('DIR::render_small1') if $MCDEBUG; + return undef unless ($small_file_size && ($root_nfs || !$root->is_remote)); $dm->_init_path; - return undef if ($dm->metalink && !$dm->accept_all) || ($dm->meta4 && !$dm->accept_all) || $dm->mirrorlist || $dm->zsync; + $c->log->error('DIR::render_small2') if $MCDEBUG; + return undef if ($dm->metalink && $dm->accept) || ($dm->meta4 && $dm->accept) || $dm->mirrorlist || $dm->zsync; + $c->log->error('DIR::render_small3') if $MCDEBUG; my ($path, undef) = $dm->path; my $full; return $root->render_file_if_small($dm, $path, $small_file_size) unless $root_nfs; + $c->log->error('DIR::render_small4') if $MCDEBUG; + my $original_path = $dm->path; + return undef if $original_path ne $path || $dm->extra; + $c->log->error($c->dumper('DIR::render_small5', $original_path, $path, $dm->extra)) if $MCDEBUG; $full = $root_nfs . $path; my $size; eval { $size = -s $full if -f $full; }; return undef unless (defined $size) && $size <= $small_file_size; - my $c = $dm->c; - $c->render_file(filepath => $full, content_type => $dm->mime); - return 1; + $c->log->error('DIR::render_small6') if $MCDEBUG; + return $root->render_file($dm, $path, 1, 1); } # if we don't render file directly - we set max-age to short value, because redirect or metalink may change diff --git a/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm b/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm index 78d821f4..12d7c3e8 100644 --- a/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm +++ b/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm @@ -150,7 +150,7 @@ sub register { } if (!$file) { - return $c->render(status => 404, text => "File not found"); + return undef; } $c->log->error($c->dumper('RENDER FILE_ID', $file->{id})) if $MCDEBUG; $c->res->headers->vary('Accept, COUNTRY'); diff --git a/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm b/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm index bf4a6ffc..a0c85675 100644 --- a/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm +++ b/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm @@ -141,10 +141,10 @@ sub is_dir { } sub render_file { - my ($self, $dm, $filepath, $not_miss) = @_; + my ($self, $dm, $filepath, $not_miss, $from_nfs) = @_; my $c = $dm->c; my $nfs = $self->rootnfs; - if ($nfs && $dm->must_render_from_root && -f $nfs . $filepath) { + if ($nfs && ($dm->must_render_from_root || $from_nfs) && -f ($nfs . $filepath)) { $c->render_file(filepath => $nfs . $filepath, content_type => $dm->mime, content_disposition => 'inline'); $c->stat->redirect_to_root($dm, $not_miss); return 1; diff --git a/t/environ/02-files-hashes.sh b/t/environ/02-files-hashes.sh index 94f09d1e..c54c176b 100755 --- a/t/environ/02-files-hashes.sh +++ b/t/environ/02-files-hashes.sh @@ -17,8 +17,18 @@ for x in $mc; do echo 1111111111 > $x/dt/folder1/file1.1.dat echo 1111111111 > $x/dt/folder1/file2.1.dat echo 2345 > $x/dt/folder1/file2.1.dat.zsync + echo 2345 > $x/dt/folder1/fileX.dat.zsync done +$mc/curl -I /download/folder1/file2.1.dat.zsync | grep '200 OK' +$mc/curl -I /download/folder1/fileX.dat.zsync | grep '200 OK' +$mc/curl -I /download/folder1/file2.1.dat.meta4 | grep '425' +$mc/curl -H 'Accept: Application/x-zsync' -I /download/folder1/file2.1.dat | grep '425' +$mc/curl -H 'Accept: Application/metalink+xml' -I /download/folder1/file2.1.dat | grep '425' +$mc/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file2.1.dat | grep '200 OK' +$mc/curl -H 'Accept: Application/metalink+xml, Application/x-zsync' -I /download/folder1/file2.1.dat | grep '425' +$mc/curl -H 'Accept: Application/metalink+xml, Application/x-zsync, */*' -I /download/folder1/file2.1.dat | grep '200 OK' + # force scan $mc/backstage/job -e folder_sync -a '["/folder1"]' $mc/backstage/shoot diff --git a/t/environ/03-headquarter-subsidiaries-hashes.sh b/t/environ/03-headquarter-subsidiaries-hashes.sh index 5161b153..cdd527f8 100755 --- a/t/environ/03-headquarter-subsidiaries-hashes.sh +++ b/t/environ/03-headquarter-subsidiaries-hashes.sh @@ -80,8 +80,12 @@ for i in 9 6 7 8; do mc$i/backstage/shoot mc$i/backstage/shoot -q hashes mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id = file_id where name='file-Media.iso'" - for x in '' .metalink .mirrorlist; do - mc$i/curl -I /folder1/file-Media.iso$x | grep -C 10 302 | grep /folder1/file4.1.dat$x | grep -v /download/folder1/file4.1.dat$x + for x in '' metalink mirrorlist; do + ext="" + par="" + [ -z "$x"] || ext=.$x + [ -z "$x"] || par=?$x + mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$par | grep -v /download/folder1/file4.1.dat done done diff --git a/t/environ/04-remote-current.sh b/t/environ/04-remote-current.sh index 2e53ba43..79a2fee4 100755 --- a/t/environ/04-remote-current.sh +++ b/t/environ/04-remote-current.sh @@ -18,8 +18,10 @@ for x in $ap7 $ap8 $ap9; do echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs -n 1 touch sha256sum $x/dt/folder1/file1.1-Media.iso > $x/dt/folder1/file1.1-Media.iso.sha256 echo 111112 > $x/dt/folder1/file2.1-Media.iso + echo 111113 > $x/dt/folder1/file2.1-Media.iso.zsync sha256sum $x/dt/folder1/file2.1-Media.iso > $x/dt/folder1/file2.1-Media.iso.sha256 ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s file1.1-Media.iso.sha256 file-Media.iso.sha256 ) + ( cd $x/dt/folder1 && ln -s file2.1-Media.iso.zsync file-Media.iso.zsync ) done for x in $ap7 $ap8 $ap9; do @@ -45,6 +47,9 @@ $mc/curl -I /download/folder1/file-Media.iso | grep -C 10 302 | grep /dow $mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256 $mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 " +$mc/curl -I /download/folder1/file2.1-Media.iso.zsync | grep --color=never -P 'Location: http://127.0.0.1:1324/folder1/file2.1-Media.iso.zsync\r$' +$mc/curl -I /download/folder1/file-Media.iso.zsync | grep --color=never -P 'file2.1-Media.iso.zsync\r$' + echo now change the symlink and make sure redirect changes ( cd $ap9/dt/folder1 diff --git a/t/environ/04-remote-nfs.sh b/t/environ/04-remote-nfs.sh index a1b5dd01..ded4d03a 100755 --- a/t/environ/04-remote-nfs.sh +++ b/t/environ/04-remote-nfs.sh @@ -26,6 +26,9 @@ ap7=$(environ ap7) for x in $ap7 $ap8 $ap9 $mc1 $mc2; do mkdir -p $x/dt/{folder1,folder2,folder3} echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch + echo 111112 > $x/dt/folder1/file2.1-Media.iso + echo 111113 > $x/dt/folder1/file2.1-Media.iso.zsync + ( cd $x/dt/folder1 && ln -s file2.1-Media.iso.zsync file-Media.iso.zsync ) mkdir -p $x/dt/updates/tool ( @@ -77,4 +80,13 @@ $mc2/backstage/shoot echo now we learned about all 3 folders $mc2/sql_test 3 == 'select count(*) from folder' +$mc2/curl -H 'Accept: Application/x-zsync' -IL /download/folder1/file-Media.iso.zsync | grep '404 Not Found' +$mc2/curl -H 'Accept: Application/x-zsync' -I /download/folder1/file2.1-Media.iso.zsync | grep '404 Not Found' # we don't have zhashes for this file +$mc2/curl -H 'Accept: Application/x-zsync' -I /download/folder1/file2.1-Media.iso | grep '404 Not Found' # we don't have zhashes for this file +$mc2/curl -H 'Accept: Application/metalink+xml' -I /download/folder1/file2.1-Media.iso | grep '200 OK' +$mc2/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file2.1-Media.iso | grep '200 OK' +$mc2/curl -H 'Accept: Application/metalink+xml, */*' -I /download/folder1/file-Media.iso.zsync | grep --color=never -P '/download/folder1/file2.1-Media.iso.zsync\r$' + +$mc2/curl -I /download/folder1/file-Media.iso.zsync | grep --color=never -P '/download/folder1/file2.1-Media.iso.zsync\r$' + echo success