Add test for platform independent resources and vanished resources

src/mirror_tarball.jl

@@ -28,7 +28,8 @@ function mirror_tarball(
         upstreams::AbstractVector,
         static_dir::AbstractString;
         http_parameters::Dict{Symbol, Any} = Dict{Symbol, Any}(),
-        packages::AbstractVector = [],
+        packages::Union{AbstractVector, Nothing} = nothing,
+        registry_hash::Union{AbstractString, Nothing} = nothing,
         show_progress = true,
 )
     ### Except for the complex error handling strategy, the mirror routine
@@ -83,7 +84,7 @@ function mirror_tarball(
 
     # 1. query latest registry hash
     upstreams = normalize_upstream.(upstreams)
-    latest_hash = query_latest_hash(registry, upstreams)
+    latest_hash = isnothing(registry_hash) ? query_latest_hash(registry, upstreams) : registry_hash
     if isnothing(latest_hash)
         @error "failed to query the latest registry hash" uuid=uuid hash=latest_hash upstreams=upstreams
         error("stop mirroring.")
@@ -99,13 +100,15 @@ function mirror_tarball(
     end
 
     # 3. read and download `/package/$uuid/$hash`
-    packages = isempty(packages) ? mktempdir() do tmpdir
-        open(tarball, "r") do io
-            Tar.extract(decompress(io), tmpdir)
+    if isnothing(packages)
+        packages = mktempdir() do tmpdir
+            open(tarball, "r") do io
+                Tar.extract(decompress(io), tmpdir)
+            end
+            # only returns packages that are not stored in static_dir
+            read_packages(tmpdir; fetch_full_registry=false, static_dir=static_dir)
         end
-        # only returns packages that are not stored in static_dir
-        read_packages(tmpdir; fetch_full_registry=false, static_dir=static_dir)
-    end : packages
+    end
 
     num_versions = mapreduce(x -> length(x.versions), +, packages)
     @info "Start mirrorring" date=now() registry=name uuid=uuid hash=latest_hash num_versions=num_versions upstreams=upstream_str
@@ -198,7 +201,11 @@ function read_records(logfile)
     isempty(records) && return Set()
 
     # skip datetime line
-    sort(collect(Set(records[2:end])))
+    if query_last_try_datetime(logfile) != DateTime(0)
+        records = records[2:end]
+    end
+
+    sort(collect(Set(records)))
 end
 
 function update_registries(registries_file, uuid, hash)

src/utils/server_utils.jl

@@ -266,11 +266,17 @@ function download_and_verify(
             try
                 timeout_call(default_http_parameters[:timeout]) do
                     while true
+                        sleep(0.1)
+
                         if any(t->istaskdone(t) && t.result === true, task_pool)
                             interrupt_task(task_pool)
                             return true
                         end
-                        sleep(0.1)
+
+                        if all(istaskdone, task_pool)
+                            @warn "fail to download resource" resource=resource tarball=tarball upstreams=join(servers, ", ")
+                            return false
+                        end
                     end
                 end
             catch err

test/tst_mirror_tarball.jl

@@ -1,6 +1,8 @@
-using StorageMirrorServer: RegistryMeta, query_latest_hash, download_and_verify, query_artifacts
+using StorageMirrorServer: RegistryMeta, query_latest_hash, download_and_verify, query_artifacts, read_packages
+using StorageMirrorServer: mirror_tarball
 using StorageMirrorServer: decompress
 using StorageMirrorServer: timeout_call
+using Tar
 
 @testset "mirror_tarball" begin
     tmp_testdir = mktempdir()
@@ -17,17 +19,23 @@ using StorageMirrorServer: timeout_call
     @test isfile(tarball)
 
     packages = mktempdir() do tmpdir
+        rst = []
         open(tarball, "r") do io
             Tar.extract(decompress(io), tmpdir)
         end
         # only returns packages that are not stored in static_dir
-        read_packages(tmpdir; fetch_full_registry=false, static_dir=tmp_testdir, latest_versions_num=1) do pkg
-            occursin("MbedTLS", pkg.name)
-        end
+        append!(rst, read_packages(tmpdir; fetch_full_registry=false, static_dir=tmp_testdir, latest_versions_num=1) do pkg
+            occursin("MbedTLS", pkg.name) # platform-dependent artifacts
+        end)
+        append!(rst, read_packages(tmpdir; fetch_full_registry=false, static_dir=tmp_testdir, latest_versions_num=1, recursive=false) do pkg
+            occursin("TestImages", pkg.name) || # platform-independent artifacts
+            occursin("StanMCMCChain", pkg.name) # vanished resources
+        end)
+        return rst
     end
 
     @time rst_hash = timeout_call(1200) do
-        mirror_tarball(registry, upstreams, tmp_testdir; packages=packages)
+        mirror_tarball(registry, upstreams, tmp_testdir; packages=packages, registry_hash=registry_hash)
     end
     @test rst_hash == registry_hash
 
@@ -36,7 +44,12 @@ using StorageMirrorServer: timeout_call
         for (ver, hash_info) in pkg.versions
             tree_hash = hash_info["git-tree-sha1"]
             tarball = joinpath(tmp_testdir, "package", pkg.uuid, tree_hash)
-            @test isfile(tarball)
+            if pkg.name == "StanMCMCChain"
+                # instead this one should be listed in "/failed_resources.txt"
+                @test !isfile(tarball)
+            else
+                @test isfile(tarball)
+            end
         end
     end
 
@@ -46,4 +59,36 @@ using StorageMirrorServer: timeout_call
 
     registries_file = joinpath(tmp_testdir, "registries")
     @test isfile(registries_file) && occursin(resource, readline(registries_file))
+
+    # test if vanished resources are listed here
+    failed_resources_log = joinpath(tmp_testdir, "failed_resources.txt")
+    @test isfile(failed_resources_log)
+    failed_records = readlines(failed_resources_log)
+    for pkg_uuid in [
+        "8f1571ae-b3a1-52af-8ab1-32258739efdb", # StanMCMCChain
+    ]
+        @test any(x->occursin(pkg_uuid, x), failed_records)
+    end
+
+    # an immediately incremental build should does nothing
+    tarball = joinpath(tmp_testdir, "registry", registry.uuid, registry_hash)
+    packages = mktempdir() do tmpdir
+        rst = []
+        open(tarball, "r") do io
+            Tar.extract(decompress(io), tmpdir)
+        end
+        # only returns packages that are not stored in static_dir
+        append!(rst, read_packages(tmpdir; fetch_full_registry=false, static_dir=tmp_testdir, latest_versions_num=1) do pkg
+            occursin("MbedTLS", pkg.name) # platform-dependent artifacts
+        end)
+        append!(rst, read_packages(tmpdir; fetch_full_registry=false, static_dir=tmp_testdir, latest_versions_num=1, recursive=false) do pkg
+            occursin("TestImages", pkg.name) || # platform-independent artifacts
+            occursin("StanMCMCChain", pkg.name) # vanished resources
+        end)
+        return rst
+    end
+    @time err_msg = timeout_call(30) do
+        @capture_err mirror_tarball(registry, upstreams, tmp_testdir; packages=packages, registry_hash=registry_hash)
+    end
+    @test occursin("$(length(packages)) previously failed resources are skipped during this build", err_msg)
 end