Merge pull request #75 from regro/dot-conda

ENH support dot conda

Author: beckermr

libcflib/harvester.py

@@ -43,64 +43,87 @@ def yaml_construct_fallback(loader, node):
 )
 
 
+def harvest_dot_conda(io_like, filename):
+    from conda_package_streaming import package_streaming
+    stream = package_streaming.stream_conda_component(
+        filename, io_like, component=package_streaming.CondaComponent.info
+    )
+    data = harvest_tarfile(stream)
+    stream.close()
+
+    return data
+
+
 def harvest(io_like):
     tf = tarfile.open(fileobj=io_like, mode="r:bz2")
-
-    # info/files
-    file_listing = tf.extractfile("info/files").readlines()
-    file_listing = (fn.decode("utf8").strip() for fn in file_listing if fn)
-    file_listing = [fn for fn in file_listing if filter_file(fn)]
-
-    # info/recipe/meta.yaml
-    try:
-        rendered_recipe = ruamel_yaml.safe_load(tf.extractfile("info/recipe/meta.yaml"))
-    except ScannerError:
-        # Non parseable
-        rendered_recipe = {}
-    except KeyError:
-        # older artifacts have a meta.yaml in another location
-        try:
-            rendered_recipe = ruamel_yaml.safe_load(tf.extractfile("info/meta.yaml"))
-        except ScannerError:
-            # Non parseable
-            rendered_recipe = {}
-
-    try:
-        raw_recipe = (
-            tf.extractfile("info/recipe/meta.yaml.template").read().decode("utf8")
-        )
-    except KeyError:
-        raw_recipe = tf.extractfile("info/recipe/meta.yaml").read().decode("utf8")
-
-    try:
-        conda_build_config = ruamel_yaml.safe_load(
-            tf.extractfile("info/recipe/conda_build_config.yaml")
-        )
-    except KeyError:
-        conda_build_config = {}
-
-    try:
-        about = json.load(tf.extractfile("info/about.json"))
-    except KeyError:
-        about = {}
-    index = json.load(tf.extractfile("info/index.json"))
+    return harvest_tarfile(tf)
+
+
+def harvest_tarfile(tf_or_stream):
+    rendered_recipe = {}
+    index = {}
+    about = {}
+    raw_recipe = ""
+    conda_build_config = {}
+    raw_recipe_backup = ""
+
+    for _data in tf_or_stream:
+        if isinstance(_data, tarfile.TarInfo):
+            mem = _data
+            tf = tf_or_stream
+        else:
+            tf, mem = _data
+
+        if mem.name == "info/files":
+            # info/files
+            file_listing = tf.extractfile(mem).readlines()
+            file_listing = [fn.decode("utf8").strip() for fn in file_listing if fn]
+            file_listing = [fn for fn in file_listing if filter_file(fn)]
+        elif mem.name == "info/recipe/meta.yaml":
+            raw_recipe_backup = tf.extractfile(mem).read(mem.size).decode("utf8")
+            # info/recipe/meta.yaml
+            try:
+                rendered_recipe = ruamel_yaml.safe_load(raw_recipe_backup)
+            except ScannerError:
+                # Non parseable
+                rendered_recipe = {}
+        elif mem.name == "info/meta.yaml":
+            # older artifacts have a meta.yaml in another location
+            try:
+                rendered_recipe = ruamel_yaml.safe_load(tf.extractfile(mem).read(mem.size))
+            except ScannerError:
+                # Non parseable
+                rendered_recipe = {}
+        elif mem.name == "info/about.json":
+            about = json.loads(tf.extractfile(mem).read(mem.size))
+        elif mem.name == "info/index.json":
+            index = json.loads(tf.extractfile(mem).read(mem.size))
+        elif mem.name == "info/recipe/meta.yaml.template":
+            raw_recipe = tf.extractfile(mem).read(mem.size).decode("utf8")
+        elif mem.name == "info/recipe/conda_build_config.yaml":
+            conda_build_config = ruamel_yaml.safe_load(tf.extractfile(mem).read(mem.size))
 
     return {
         "metadata_version": METADATA_VERSION,
-        "name": index["name"],
-        "version": index["version"],
+        "name": index.get("name", ""),
+        "version": index.get("version", ""),
         "index": index,
         "about": about,
         "rendered_recipe": rendered_recipe,
-        "raw_recipe": raw_recipe,
+        "raw_recipe": raw_recipe if len(raw_recipe) > 0 else raw_recipe_backup,
         "conda_build_config": conda_build_config,
         "files": file_listing,
     }
 
 
 def harvest_from_filename(filename):
     with open(filename, "rb") as fo:
-        return harvest(fo)
+        if filename.endswith(".tar.bz2"):
+            return harvest(fo)
+        elif filename.endswith(".conda"):
+            return harvest_dot_conda(fo, filename)
+        else:
+            raise RuntimeError(f"File '{filename}' is not a recognized conda format!")
 
 
 if __name__ == "__main__":

libcflib/models.xsh

@@ -113,7 +113,7 @@ class Artifact(Model):
         arch : str or None, optional
             Architeture name
         name : str or None, optional
-            Fully resolved name, without *.tar.bz2 or *.json
+            Fully resolved name, without *.conda, *.tar.bz2 or *.json
         path : str or None, optional
             Path to artifact file. This is relative to the libcfgraph repo dir.
         """
@@ -129,6 +129,8 @@ class Artifact(Model):
                 name = name[:-8]
             elif name.endswith(".json"):
                 name = name[:-5]
+            elif name.endswith(".conda"):
+                name - name[:-6]
             self._path = os.path.join(pkg, channel, arch, name + ".json")
         else:
             msg = "Artifact path or pkg, channel, arch and name must be given. "

libcflib/preloader.py

@@ -17,7 +17,7 @@
 import requests
 import tqdm
 
-from .harvester import harvest
+from .harvester import harvest, harvest_dot_conda
 from .tools import expand_file_and_mkdirs
 
 
@@ -43,6 +43,12 @@ def fetch_arch(arch):
             ".tar.bz2", ".json"
         )
         yield v["name"], file_name, package_url
+    for p, v in repodata["packages.conda"].items():
+        package_url = f"{arch}/{p}"
+        file_name = package_url.replace("https://conda.anaconda.org/", "").replace(
+            ".conda", ".json"
+        )
+        yield v["name"], file_name, package_url
 
 
 def fetch_upstream() -> Dict[str, Dict[str, str]]:
@@ -110,8 +116,14 @@ def reap_package(root_path, package, dst_path, src_url, progress_callback=None):
                 shell=True,
                 check=True,
             )
-            with open(os.path.join(tmpdir, os.path.basename(src_url)), "rb") as filelike:
-                harvested_data = harvest(filelike)
+            pkg_pth = os.path.join(tmpdir, os.path.basename(src_url))
+            with open(pkg_pth, "rb") as filelike:
+                if pkg_pth.endswith(".tar.bz2"):
+                    harvested_data = harvest(filelike)
+                elif pkg_pth.endswith(".conda"):
+                    harvested_data = harvest_dot_conda(filelike, pkg_pth)
+                else:
+                    raise RuntimeError(f"File '{pkg_pth}' is not a recognized conda format!")
         with open(
             expand_file_and_mkdirs(os.path.join(root_path, package, dst_path)), "w"
         ) as fo:

requirements/run.txt

@@ -13,3 +13,4 @@ networkx
 requests
 ruamel_yaml
 wget
+conda-package-streaming