tests, simpleget: replace use of test-snapd-curl with simpleget where possible, support request headers

tests/core/snapd-refresh-vs-services-reboots/task.yaml

@@ -27,19 +27,17 @@ environment:
   SNAPD_2_49_2_ARMHF: https://storage.googleapis.com/snapd-spread-tests/snaps/snapd_2.49.2_11586.snap
 
 prepare: |
-  snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
-
   # save the version of snapd from the PR to refresh to later
   INITIAL_REV=$(snap list snapd | tail -n +2 | awk '{print $3}')
   cp "/var/lib/snapd/snaps/snapd_$INITIAL_REV.snap" snapd-pr.snap
 
   # download and install snapd 2.49.2
   if os.query is-pc-amd64; then
-    test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_X86"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_X86"
   elif os.query is-arm64; then
-    test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARM64"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARM64"
   elif os.query is-armhf; then
-    test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARMHF"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARMHF"
   else
     echo "architecture not supported for this variant"
     exit 0

tests/core/snapd-refresh-vs-services/task.yaml

@@ -42,8 +42,6 @@ environment:
   SNAPD_2_49_2_ARMHF: https://storage.googleapis.com/snapd-spread-tests/snaps/snapd_2.49.2_11586.snap
 
 prepare: |
-  # install http snap to download files
-  snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
   # save the current version of snapd for later
   INITIAL_REV=$(snap list snapd | tail -n +2 | awk '{print $3}')
   cp "/var/lib/snapd/snaps/snapd_$INITIAL_REV.snap" snapd-pr.snap
@@ -81,11 +79,11 @@ execute: |
   # introduced so we can install a snap service that will not have Requires= in
   # it
   if os.query is-pc-amd64; then
-    test-snapd-curl.curl -s -o snapd_2.49.1.snap "$SNAPD_2_49_1_X86"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.1.snap "$SNAPD_2_49_1_X86"
   elif os.query is-arm64; then
-    test-snapd-curl.curl -s -o snapd_2.49.1.snap "$SNAPD_2_49_1_ARM64"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.1.snap "$SNAPD_2_49_1_ARM64"
   elif os.query is-armhf; then
-    test-snapd-curl.curl -s -o snapd_2.49.1.snap "$SNAPD_2_49_1_ARMHF"
+    "$TESTSTOOLS/simpleget" -o snapd_2.49.1.snap "$SNAPD_2_49_1_ARMHF"
   fi
 
   snap install --dangerous snapd_2.49.1.snap
@@ -112,11 +110,11 @@ execute: |
   elif [ "${SNAPD_VERSION_UNDER_TEST}" = "2.49.2" ]; then
     # download and install snapd 2.49.2
     if os.query is-pc-amd64; then
-      test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_X86"
+      "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_X86"
     elif os.query is-arm64; then
-      test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARM64"
+      "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARM64"
     elif os.query is-armhf; then
-      test-snapd-curl.curl -s -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARMHF"
+      "$TESTSTOOLS/simpleget" -o snapd_2.49.2.snap "$SNAPD_2_49_2_ARMHF"
     fi
 
     echo "Refreshing snapd to 2.49.2"

tests/lib/tools/simpleget

@@ -13,6 +13,7 @@ from urllib.parse import urlparse
 def parse_arguments() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="simple file getter")
     parser.add_argument("-o", "--output", help="output file name")
+    parser.add_argument("-H", "--header", action="append", help="set a header")
     parser.add_argument("URL", help="download URL")
     return parser.parse_args()
 
@@ -41,17 +42,50 @@ def main() -> None:
     # create a temp in the same directory as output
     outdir = os.path.dirname(output)
 
+    headers = {}
+    if opts.header:
+        for hdr in opts.header:
+            h, _, v = hdr.partition(": ")
+            if v == "":
+                raise RuntimeError("invalid header {}".format(hdr))
+            headers[h] = v
+
+    req = request.Request(opts.URL, headers=headers)
     with tempfile.NamedTemporaryFile(mode="wb", dir=outdir, delete=False) as outf:
         name = outf.name
-        outf.close()
+
         try:
             now = datetime.datetime.now()
-            fn, _ = request.urlretrieve(
-                opts.URL, filename=outf.name, reporthook=_report
-            )
+            # recreate what urlretrieve() does
+            with request.urlopen(req) as rsp:
+                bsize = 8 * 1024
+                bcount = 0
+                read = 0
+                size = -1
+
+                if "Content-Length" in rsp.headers:
+                    size = int(rsp.headers["Content-Length"])
+
+                _report(bcount, bsize, size)
+                while True:
+                    data = rsp.read(bsize)
+                    if len(data) == 0:
+                        # done
+                        break
+
+                    read += len(data)
+                    outf.write(data)
+                    _report(bcount, bsize, size)
+
+                if size > 0 and read < size:
+                    raise request.ContentTooShortError(
+                        "got {} out of {} bytes".format(read, size),
+                        (),
+                    )
+
             after = datetime.datetime.now()
 
-            os.rename(fn, output)
+            os.rename(name, output)
         except:
             os.unlink(name)
             raise

tests/main/default-tracks/task.yaml

@@ -14,12 +14,9 @@ environment:
     A_SNAP: test-snapd-default-track
     A_TRACK: default
 
-prepare: |
-    snap install test-snapd-curl --edge
-
 execute: |
     # first, precondition check that the snap has a default track
-    snap run test-snapd-curl.curl -H "Snap-Device-Series: 16" "https://api.snapcraft.io/v2/snaps/info/$A_SNAP" > info
+    "$TESTSTOOLS/simpleget" -H "Snap-Device-Series: 16" "https://api.snapcraft.io/v2/snaps/info/$A_SNAP" -o info
     test "$( gojq -r '."default-track"' < info )" == "$A_TRACK"
 
     # TODO: check the output of 'snap info' for the default-track-having snap
@@ -33,7 +30,6 @@ execute: |
     "$TESTSTOOLS"/snapd-state check-state ".data.snaps.\"$A_SNAP\".channel" = "$A_TRACK/candidate"
 
     snap remove --purge "$A_SNAP"
-    snap remove --purge test-snapd-curl
 
     # now try a multi-install
     snap install "$A_SNAP" "test-snapd-tools"

tests/nested/manual/cloud-init-never-used-not-vuln/task.yaml

@@ -35,18 +35,16 @@ prepare: |
     # thus this test would no longer be testing that old vulnerable devices 
     # become safe after refreshing to the fix
     if [ "$NESTED_BUILD_SNAPD_FROM_CURRENT" = "false" ]; then
-        snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
-
         if os.query is-xenial; then
             # uc16 uses core snap
-            test-snapd-curl.curl -s -o core_2.45.snap "$SNAPD_2_45_CORE_SNAP"
+            "$TESTSTOOLS/simpleget" -o core_2.45.snap "$SNAPD_2_45_CORE_SNAP"
             # The core snap is unpacked and repacked to prevent it is auto-refreshed when
             # it is installed with --dangerous as it has a different hash
             unsquashfs -d core-snap core_2.45.snap
             snap pack core-snap/ "$(tests.nested get extra-snaps-path)"
         else 
             # uc18 uses snapd snap
-            test-snapd-curl.curl -s -o snapd_2.45.snap "$SNAPD_2_45_SNAPD_SNAP"
+            "$TESTSTOOLS/simpleget" -o snapd_2.45.snap "$SNAPD_2_45_SNAPD_SNAP"
             # The snapd snap is unpacked and repacked to prevent it from being
             # auto-refreshed when it is installed
             unsquashfs -d snapd-snap snapd_2.45.snap

tests/nested/manual/cloud-init-nocloud-not-vuln/task.yaml

@@ -41,18 +41,16 @@ prepare: |
     # thus this test would no longer be testing that old vulnerable devices 
     # become safe after refreshing to the fix
     if [ "$NESTED_BUILD_SNAPD_FROM_CURRENT" = "false" ]; then
-        snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
-
         if os.query is-xenial; then
             # uc16 uses core snap
-            test-snapd-curl.curl -s -o core_2.45.snap "$SNAPD_2_45_CORE_SNAP"
+            "$TESTSTOOLS/simpleget" -o core_2.45.snap "$SNAPD_2_45_CORE_SNAP"
             # The core snap is unpacked and repacked to prevent it is auto-refreshed when
             # it is installed with --dangerous as it has a different hash
             unsquashfs -d core-snap core_2.45.snap
             snap pack core-snap/ "$(tests.nested get extra-snaps-path)"
         else 
             # uc18 uses snapd snap
-            test-snapd-curl.curl -s -o snapd_2.45.snap "$SNAPD_2_45_SNAPD_SNAP"
+            "$TESTSTOOLS/simpleget" -o snapd_2.45.snap "$SNAPD_2_45_SNAPD_SNAP"
             # repack to prevent automatic refresh
             unsquashfs -d snapd-snap snapd_2.45.snap
             snap pack snapd-snap/ "$(tests.nested get extra-snaps-path)"