order caffe2 ubuntu configs contiguously (pytorch#17427)

Summary:
This involves another purely cosmetic (ordering) change to the `config.yml` to facilitate simpler logic.

Other changes:
* add some review feedback as comments
* exit with nonzero status on config.yml mismatch
* produce a diagram for pytorch builds
Pull Request resolved: pytorch#17427

Differential Revision: D14197618

Pulled By: kostmo

fbshipit-source-id: 267439d3aa4c0a80801adcde2fa714268865900e

Author: kostmo

.circleci/README.md

@@ -24,3 +24,13 @@ Furthermore, consistency is enforced within the YAML config itself, by using a s
 multiple parts of the file.
 
 See https://github.com/pytorch/pytorch/issues/17038
+
+
+Future direction
+----------------
+
+### Declaring sparse config subsets
+See comment [here](https://github.com/pytorch/pytorch/pull/17323#pullrequestreview-206945747):
+
+In contrast with a full recursive tree traversal of configuration dimensions,
+> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.

.circleci/cimodel/binary_build_definitions.py

@@ -91,18 +91,18 @@ def gen_yaml_tree(self, build_or_test):
         return d
 
 
-def get_root(smoke):
+def get_root(smoke, name):
 
     return make_build_configs.TopLevelNode(
-        "Builds",
+        name,
         make_build_configs.CONFIG_TREE_DATA,
         smoke,
     )
 
 
 def gen_build_env_list(smoke):
 
-    root = get_root(smoke)
+    root = get_root(smoke, "N/A")
     config_list = conf_tree.dfs(root)
 
     newlist = []
@@ -214,7 +214,7 @@ def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule):
 
     jobs_dict[toplevel_key] = d
 
-    graph = visualization.generate_graph(get_root(smoke))
+    graph = visualization.generate_graph(get_root(smoke, toplevel_key))
     graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi")
 
 

.circleci/cimodel/caffe2_build_definitions.py

@@ -12,27 +12,22 @@
 DOCKER_IMAGE_VERSION = 248
 
 
-# TODO Sort the config.yml upstream so the ubuntu configs are contiguous
 CONFIG_HIERARCHY = [
+    (Ver("ubuntu", "14.04"), [
+        (Ver("gcc", "4.8"), ["py2"]),
+        (Ver("gcc", "4.9"), ["py2"]),
+    ]),
     (Ver("ubuntu", "16.04"), [
+        (Ver("cuda", "8.0"), ["py2"]),
         (Ver("cuda", "9.0"), [
+            # TODO make explicit that this is a "secret TensorRT build"
+            #  (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
             "py2",
             "cmake",
         ]),
         (Ver("cuda", "9.1"), ["py2"]),
         (Ver("mkl"), ["py2"]),
-    ]),
-    (Ver("ubuntu", "14.04"), [
-        (Ver("gcc", "4.8"), ["py2"]),
-    ]),
-    (Ver("ubuntu", "16.04"), [
         (Ver("gcc", "5"), ["onnx_py2"]),
-        (Ver("cuda", "8.0"), ["py2"]),
-    ]),
-    (Ver("ubuntu", "14.04"), [
-        (Ver("gcc", "4.9"), ["py2"]),
-    ]),
-    (Ver("ubuntu", "16.04"), [
         (Ver("clang", "3.8"), ["py2"]),
         (Ver("clang", "3.9"), ["py2"]),
         (Ver("clang", "7"), ["py2"]),
@@ -42,6 +37,8 @@
         (Ver("cuda", "9.0"), ["py2"]),
     ]),
     (Ver("macos", "10.13"), [
+        # TODO ios and system aren't related. system qualifies where the python comes
+        #  from (use the system python instead of homebrew or anaconda)
         (Ver("ios"), ["py2"]),
         (Ver("system"), ["py2"]),
     ]),
@@ -65,6 +62,7 @@ def is_build_only(self):
             "android",
         ] or self.get_platform() == "macos"
 
+    # TODO: Eventually we can probably just remove the cudnn7 everywhere.
     def get_cudnn_insertion(self):
 
         omit = self.language == "onnx_py2" \
@@ -128,7 +126,7 @@ def gen_yaml_tree(self):
             tuples.append(("BUILD_IOS", miniutils.quote("1")))
 
         if self.phase == "test":
-            use_cuda_docker = str(self.compiler) not in ["mkl", "gcc4.8", "gcc5"]
+            use_cuda_docker = self.compiler.name == "cuda"
             if use_cuda_docker:
                 tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
 
@@ -139,6 +137,8 @@ def gen_yaml_tree(self):
             if not self.distro.name == "macos":
                 tuples.append(("BUILD_ONLY", miniutils.quote("1")))
 
+        # TODO: not sure we need the distinction between system and homebrew anymore. Our python handling in cmake
+        #  and setuptools is more robust now than when we first had these.
         if self.distro.name == "macos":
             tuples.append(("PYTHON_INSTALLATION", miniutils.quote("system")))
             tuples.append(("PYTHON_VERSION", miniutils.quote("2")))
@@ -150,7 +150,7 @@ def gen_yaml_tree(self):
         ])
 
         if self.phase == "test":
-            is_large = str(self.compiler) in ["mkl", "gcc4.8"] or self.language == "onnx_py2"
+            is_large = self.compiler.name != "cuda"
 
             resource_class = "large" if is_large else "gpu.medium"
             d["resource_class"] = resource_class
@@ -187,6 +187,7 @@ def get_caffe2_workflows():
     configs = gen_build_list()
 
     # TODO Why don't we build this config?
+    # See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540
     filtered_configs = filter(lambda x: not (str(x.distro) == "ubuntu14.04" and str(x.compiler) == "gcc4.9"), configs)
 
     x = []

.circleci/cimodel/conf_tree.py

@@ -20,11 +20,7 @@ def __init__(self, parent, node_name):
         self.props = {}
 
     def get_label(self):
-        label = self.node_name
-        if not label:
-            # FIXME this shouldn't be necessary
-            label = "<None>"
-        return label
+        return self.node_name
 
     def get_children(self):
         return []