refactor upload loops. Refactor recursive tests.

conda_build/build.py

@@ -1428,7 +1428,6 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False,
             subprocess.call('del /s /q "{0}\\*.*" >nul 2>&1'.format(trash_dir), shell=True)
         # delete_trash(None)
 
-    already_built = set()
     extra_help = ""
     built_packages = []
     retried_recipes = []
@@ -1493,10 +1492,6 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False,
             add_recipes = []
             # add the failed one back in at the beginning - but its deps may come before it
             recipe_list.extendleft([recipe])
-            if recipe in retried_recipes:
-                raise RuntimeError("Can't build {0} due to environment creation error:\n"
-                                    .format(recipe) + error_str + "\n" + extra_help)
-            retried_recipes.append(recipe)
             for line in error_str.splitlines():
                 if not line.startswith('  - '):
                     continue
@@ -1524,46 +1519,52 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False,
                 else:
                     raise RuntimeError("Can't build {0} due to unsatisfiable dependencies:\n{1}"
                                        .format(recipe, e.pkgs) + "\n\n" + extra_help)
+            if retried_recipes.count(recipe) >= len(metadata.ms_depends('build')):
+                raise RuntimeError("Can't build {0} due to environment creation error:\n"
+                                    .format(recipe) + error_str + "\n" + extra_help)
+            retried_recipes.append(recipe)
             recipe_list.extendleft(add_recipes)
 
-        # outputs message, or does upload, depending on value of args.anaconda_upload
-        if post in [True, None]:
-            for f in built_packages:
-                # TODO: could probably use a better check for pkg type than this...
-                if f.endswith('.tar.bz2'):
-                    handle_anaconda_upload(f, config=config)
-                elif f.endswith('.whl'):
-                    handle_pypi_upload(f, config=config)
-                already_built.add(f)
-
+    if post in [True, None]:
+        # TODO: could probably use a better check for pkg type than this...
+        tarballs = [f for f in built_packages if f.endswith('.tar.bz2')]
+        wheels = [f for f in built_packages if f.endswith('.whl')]
+        handle_anaconda_upload(tarballs, config=config)
+        handle_pypi_upload(wheels, config=config)
     return built_packages
 
 
-def handle_anaconda_upload(path, config):
+def handle_anaconda_upload(paths, config):
     from conda_build.os_utils.external import find_executable
 
+    paths = utils.ensure_list(paths)
+
     upload = False
     # this is the default, for no explicit argument.
     # remember that anaconda_upload takes defaults from condarc
     if config.anaconda_upload is None:
         pass
+    elif config.token or config.user:
+        upload = True
     # rc file has uploading explicitly turned off
     elif config.anaconda_upload is False:
         print("# Automatic uploading is disabled")
+        upload = False
     else:
         upload = True
 
-    if config.token or config.user:
-        upload = True
-
     no_upload_message = """\
-# If you want to upload this package to anaconda.org later, type:
-#
-# $ anaconda upload %s
-#
+# If you want to upload package(s) to anaconda.org later, type:
+
+"""
+    for package in paths:
+        no_upload_message += "anaconda upload {}\n".format(package)
+
+    no_upload_message += """\
+
 # To have conda build upload to anaconda.org automatically, use
 # $ conda config --set anaconda_upload yes
-""" % path
+"""
     if not upload:
         print(no_upload_message)
         return
@@ -1576,23 +1577,23 @@ def handle_anaconda_upload(path, config):
 # Try:
 # $ conda install anaconda-client
 ''')
-    print("Uploading to anaconda.org")
     cmd = [anaconda, ]
 
     if config.token:
         cmd.extend(['--token', config.token])
     cmd.extend(['upload', '--force'])
     if config.user:
         cmd.extend(['--user', config.user])
-    cmd.append(path)
-    try:
-        subprocess.call(cmd)
-    except:
-        print(no_upload_message)
-        raise
+    for package in paths:
+        try:
+            print("Uploading {} to anaconda.org".format(os.path.basename(package)))
+            subprocess.call(cmd + [package])
+        except subprocess.CalledProcessError:
+            print(no_upload_message)
+            raise
 
 
-def handle_pypi_upload(f, config):
+def handle_pypi_upload(wheels, config):
     args = ['twine', 'upload', '--sign-with', config.sign_with, '--repository', config.repository]
     if config.user:
         args.extend(['--user', config.user])
@@ -1607,12 +1608,14 @@ def handle_pypi_upload(f, config):
     if config.repository:
         args.extend(['--repository', config.repository])
 
-    args.append(f)
-    try:
-        utils.check_call_env(args)
-    except:
-        logging.getLogger(__name__).warn("wheel upload failed - is twine installed?"
-                                         "  Is this package registered?")
+    wheels = utils.ensure_list(wheels)
+
+    for f in wheels:
+        try:
+            utils.check_call_env(args + [f])
+        except:
+            logging.getLogger(__name__).warn("wheel upload failed - is twine installed?"
+                                            "  Is this package registered?")
 
 
 def print_build_intermediate_warning(config):

tests/test_api_build.py

@@ -84,23 +84,22 @@ def test_recipe_builds(recipe, test_config, testing_workdir, monkeypatch):
     api.build(recipe, config=test_config)
 
 
-def test_token_upload(testing_workdir):
+def test_token_upload(testing_workdir, test_metadata):
     folder_uuid = uuid.uuid4().hex
     # generated with conda_test_account user, command:
     #    anaconda auth --create --name CONDA_BUILD_UPLOAD_TEST --scopes 'api repos conda'
-    args = AnacondaClientArgs(specs="conda_test_account/empty_sections_" + folder_uuid,
-                              token="co-79de533f-926f-4e5e-a766-d393e33ae98f",
+    args = AnacondaClientArgs(specs="conda_build_test/test_token_upload_" + folder_uuid,
+                              token="co-143399b8-276e-48db-b43f-4a3de839a024",
                               force=True)
 
     with pytest.raises(NotFound):
         show.main(args)
 
-    metadata, _, _ = api.render(empty_sections, activate=False)
-    metadata.meta['package']['name'] = '_'.join([metadata.name(), folder_uuid])
-    metadata.config.token = args.token
+    test_metadata.meta['package']['name'] = '_'.join([test_metadata.name(), folder_uuid])
+    test_metadata.config.token = args.token
 
     # the folder with the test recipe to upload
-    api.build(metadata)
+    api.build(test_metadata)
 
     # make sure that the package is available (should raise if it doesn't)
     show.main(args)
@@ -870,9 +869,16 @@ def test_env_creation_fail_exits_build(test_config):
 
 
 @pytest.mark.serial
-def test_recursion(test_config):
-    recipe = os.path.join(metadata_dir, '_recursive-build-two-layer')
-    api.build(recipe, config=test_config)
+def test_recursion_packages(test_config):
+    """Two packages that need to be built are listed in the recipe
 
+    make sure that both get built before the one needing them gets built."""
     recipe = os.path.join(metadata_dir, '_recursive-build-two-packages')
     api.build(recipe, config=test_config)
+
+
+@pytest.mark.serial
+def test_recursion_layers(test_config):
+    """go two 'hops' - try to build a, but a needs b, so build b first, then come back to a"""
+    recipe = os.path.join(metadata_dir, '_recursive-build-two-layer')
+    api.build(recipe, config=test_config)