more reliable retries and fallback mirror usage - #1336

source/dub/commandline.d

@@ -223,7 +223,20 @@ int runDubCommandLine(string[] args)
 			dub.defaultPlacementLocation = options.placementLocation;
 		} else {
 			// initialize DUB
-			auto package_suppliers = options.registry_urls.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))).array;
+			auto package_suppliers = options.registry_urls
+				.map!((url) {
+					// Allow to specify fallback mirrors as space separated urls. Undocumented as we
+					// should simply retry over all registries instead of using a special
+					// FallbackPackageSupplier.
+					auto urls = url.splitter(' ');
+					PackageSupplier ps = new RegistryPackageSupplier(URL(urls.front));
+					urls.popFront;
+					if (!urls.empty)
+						ps = new FallbackPackageSupplier(ps,
+							urls.map!(u => cast(PackageSupplier) new RegistryPackageSupplier(URL(u))).array);
+					return ps;
+				})
+				.array;
 			dub = new Dub(options.root_path, package_suppliers, options.skipRegistry);
 			dub.dryRun = options.annotate;
 			dub.defaultPlacementLocation = options.placementLocation;

source/dub/dub.d

@@ -757,12 +757,26 @@ class Dub {
 			return m_packageManager.getPackage(packageId, ver, dstpath);
 		}
 
-		auto path = getTempFile(packageId, ".zip");
-		supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
-		scope(exit) std.file.remove(path.toNativeString());
+		// repeat download on corrupted zips, see #1336
+		foreach_reverse (i; 0..3)
+		{
+			import std.zip : ZipException;
+
+			auto path = getTempFile(packageId, ".zip");
+			supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
+			scope(exit) std.file.remove(path.toNativeString());
+			logDiagnostic("Placing to %s...", placement.toNativeString());
 
-		logDiagnostic("Placing to %s...", placement.toNativeString());
-		return m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
+			try {
+				return m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
+			} catch (ZipException e) {
+				logInfo("Failed to extract zip archive for %s %s...", packageId, ver);
+				// rethrow the exception at the end of the loop
+				if (i == 0)
+					throw e;
+			}
+		}
+		assert(0, "Should throw a ZipException instead.");
 	}
 
 	/** Removes a specific locally cached package.

source/dub/internal/utils.d

@@ -265,7 +265,15 @@ void download(string url, string filename, uint timeout = 8)
 			}
 		}
 		else
+		{
 			std.net.curl.download(url, filename, conn);
+			// workaround https://issues.dlang.org/show_bug.cgi?id=18318
+			auto sl = conn.statusLine;
+			logDebug("Download %s %s", url, sl);
+			if (sl.code / 100 != 2)
+				throw new HTTPStatusException(sl.code,
+					"Downloading %s failed with %d (%s).".format(url, sl.code, sl.reason));
+		}
 	} else version (Have_vibe_d_http) {
 		import vibe.inet.urltransfer;
 		vibe.inet.urltransfer.download(url, filename);

test/.gitignore

@@ -11,3 +11,5 @@
 custom-unittest/custom-unittest
 path-subpackage-ref/test
 subpackage-ref/test
+
+/test_registry

test/ddox.sh.min_frontend

@@ -1 +1 @@
-2.071
+2.072

test/fetchzip.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+DIR=$(dirname "${BASH_SOURCE[0]}")
+
+. "$DIR"/common.sh
+
+PORT=$(($$ + 1024)) # PID + 1024
+
+dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null || true
+
+"$DUB" build --single "$DIR"/test_registry.d
+"$DIR"/test_registry --folder="$DIR/issue1336-registry" --port=$PORT &
+PID=$!
+sleep 0.2
+trap 'kill $PID 2>/dev/null || true' exit
+
+echo "Trying to download gitcompatibledubpackage (1.0.4)"
+timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.4 --skip-registry=all --registry=http://localhost:$PORT
+if [ $? -eq 124 ]; then
+    die 'Fetching from responsive registry should not time-out.'
+fi
+dub remove gitcompatibledubpackage --non-interactive --version=1.0.4
+
+echo "Downloads should be retried when the zip is corrupted - gitcompatibledubpackage (1.0.3)"
+zipOut=$(! timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.3 --skip-registry=all --registry=http://localhost:$PORT 2>&1)
+rc=$?
+
+if ! zipCount=$(grep -Fc 'Failed to extract zip archive' <<<"$zipOut") || [ "$zipCount" -lt 3 ] ; then
+    echo '========== +Output was ==========' >&2
+    echo "$zipOut" >&2
+    echo '========== -Output was ==========' >&2
+    die 'DUB should have tried to download the zip archive multiple times.'
+elif [ $rc -eq 124 ]; then
+    die 'DUB timed out unexpectedly.'
+fi
+if dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null; then
+    die 'DUB should not have installed a broken package.'
+fi
+
+echo "HTTP status errors on downloads should be retried - gitcompatibledubpackage (1.0.2)"
+retryOut=$(! timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.2 --skip-registry=all --registry=http://localhost:$PORT --vverbose 2>&1)
+rc=$?
+if ! retryCount=$(echo "$retryOut" | grep -Fc 'Bad Gateway') || [ "$retryCount" -lt 3 ] ; then
+    echo '========== +Output was ==========' >&2
+    echo "$retryOut" >&2
+    echo '========== -Output was ==========' >&2
+    die "DUB should have retried download on server error multiple times, but only tried $retryCount times."
+elif [ $rc -eq 124 ]; then
+    die 'DUB timed out unexpectedly.'
+fi
+if dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null; then
+    die 'DUB should not have installed a package.'
+fi
+
+echo "HTTP status errors on downloads should retry with fallback mirror - gitcompatibledubpackage (1.0.2)"
+timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.2 --skip-registry=all --registry="http://localhost:$PORT http://localhost:$PORT/fallback"
+if [ $? -eq 124 ]; then
+    die 'Fetching from responsive registry should not time-out.'
+fi
+dub remove gitcompatibledubpackage --non-interactive --version=1.0.2

test/fetchzip.sh.min_frontend

@@ -0,0 +1 @@
+2.076

test/issue1336-registry/.gitignore

@@ -0,0 +1,14 @@
+.dub
+docs.json
+__dummy.html
+docs/
+issue1336-registry.so
+issue1336-registry.dylib
+issue1336-registry.dll
+issue1336-registry.a
+issue1336-registry.lib
+issue1336-registry-test-*
+*.exe
+*.o
+*.obj
+*.lst

test/issue1336-registry/packages/gitcompatibledubpackage.json

@@ -0,0 +1 @@
+{"repository":{"project":"gitcompatibledubpackage","owner":"dlang-community","kind":"github"},"dateAdded":"2017-02-11T21:00:16","name":"gitcompatibledubpackage","categories":[],"owner":"5148973d2179ddb20b0002fd","id":"589f6d501c18646d0ee49f12","versions":[{"packageDescriptionFile":"dub.sdl","configurations":[{"name":"exe","targetType":"executable","mainSourceFile":"hello.d"},{"excludedSourceFiles":["hello.d"],"name":"lib","targetType":"library"}],"commitID":"9e3972be4c63790c32257220f40c0af7dc41bec5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"~master","license":"public domain","readmeFile":"/README.md","date":"2017-09-18T18:18:35Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]},{"packageDescriptionFile":"dub.json","commitID":"b62466d32dd6bbb0d45f9a73c91142205d1048e5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.1","license":"public domain","readmeFile":"/README.md","date":"2015-10-31T11:09:14Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite."},{"packageDescriptionFile":"dub.json","commitID":"cef89b5513a140b1e3417809f59d1957ddaad837","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.2","license":"public domain","readmeFile":"/README.md","date":"2015-10-31T11:23:53Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]},{"packageDescriptionFile":"dub.json","commitID":"d36e49d31b26ea76e1670e9d5cd89aaac40fde75","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.3","targetType":"executable","license":"public domain","readmeFile":"/README.md","date":"2017-04-05T22:12:48Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","mainSourceFile":"hello.d","sourcePaths":["."]},{"packageDescriptionFile":"dub.sdl","configurations":[{"name":"exe","targetType":"executable","mainSourceFile":"hello.d"},{"excludedSourceFiles":["hello.d"],"name":"lib","targetType":"library"}],"commitID":"9e3972be4c63790c32257220f40c0af7dc41bec5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.4","license":"public domain","readmeFile":"/README.md","date":"2017-09-18T18:18:35Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]}]}

test/issue1336-registry/packages/gitcompatibledubpackage/1.0.2.zip

@@ -0,0 +1 @@
+1.0.4.zip

test/issue1336-registry/packages/gitcompatibledubpackage/1.0.3.zip

@@ -0,0 +1 @@
+BROKEN ZIP

test/test_registry.d

@@ -0,0 +1,23 @@
+/+dub.sdl:
+dependency "vibe-d" version="~>0.8.3-alpha.1"
+versions "VibeNoSSL"
++/
+
+void main(string[] args)
+{
+	import std.conv, vibe.d;
+	immutable folder = readRequiredOption!string("folder", "Folder to service files from.");
+	immutable port = readRequiredOption!uint("port", "Port to use");
+	auto router = new URLRouter;
+	router.get("stop", (HTTPServerRequest req, HTTPServerResponse res){
+		res.writeVoidBody;
+		exitEventLoop();
+	});
+	router.get("/packages/gitcompatibledubpackage/1.0.2.zip", (req, res) {
+		res.writeBody("", HTTPStatus.badGateway);
+	});
+	router.get("*", folder.serveStaticFiles);
+	router.get("/fallback/*", folder.serveStaticFiles(new HTTPFileServerSettings("/fallback")));
+	listenHTTP(text(":", port), router);
+	runApplication();
+}

travis-ci.sh

@@ -19,11 +19,13 @@ if [ "$COVERAGE" = true ]; then
     DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh
     deactivate
     git clean -dxf -- test
-    source $(~/dlang/install.sh ldc --activate)
+    export FRONTEND=2.077
+    source $(~/dlang/install.sh ldc-1.7.0 --activate)
     DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh
     deactivate
     git clean -dxf -- test
-    source $(~/dlang/install.sh gdc --activate)
+    export FRONTEND=2.068
+    source $(~/dlang/install.sh gdc-4.8.5 --activate)
     DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh
 else
     ./build.sh