Merge pull request #1945 from csarofeen/master_merge_0828

Master merge 0828

.circleci/cimodel/data/dimensions.py

@@ -4,6 +4,7 @@
  "102",
  "113",
  "116",
+ "117",
 ]
 
 ROCM_VERSIONS = [

.circleci/cimodel/data/simple/ios_definitions.py

@@ -11,7 +11,7 @@ def __init__(self, name, custom_build_name=""):
 
  def render(self):
  extra_parts = [self.custom_build_name] if len(self.custom_build_name) > 0 else []
- return "_".join([self.name] + extra_parts)
+ return "-".join([self.name] + extra_parts).replace("_", "-")
 
 
 def get_platform(arch_variant_name):
@@ -25,30 +25,25 @@ def __init__(self, xcode_version, arch_variant, is_org_member_context=True, extr
  self.is_org_member_context = is_org_member_context
  self.extra_props = extra_props
 
- def gen_name_parts(self, with_version_dots):
-
- version_parts = self.xcode_version.render_dots_or_parts(with_version_dots)
- build_variant_suffix = "_".join([self.arch_variant.render(), "build"])
-
+ def gen_name_parts(self):
+ version_parts = self.xcode_version.render_dots_or_parts("-")
+ build_variant_suffix = self.arch_variant.render()
  return [
- "pytorch",
  "ios",
  ] + version_parts + [
  build_variant_suffix,
  ]
 
  def gen_job_name(self):
- return "_".join(self.gen_name_parts(False))
+ return "-".join(self.gen_name_parts())
 
  def gen_tree(self):
-
  platform_name = get_platform(self.arch_variant.name)
-
  props_dict = {
- "build_environment": "-".join(self.gen_name_parts(True)),
+ "name": self.gen_job_name(),
+ "build_environment": self.gen_job_name(),
  "ios_arch": self.arch_variant.name,
  "ios_platform": platform_name,
- "name": self.gen_job_name(),
  }
 
  if self.is_org_member_context:
@@ -63,16 +58,12 @@ def gen_tree(self):
 WORKFLOW_DATA = [
  IOSJob(XCODE_VERSION, ArchVariant("x86_64"), is_org_member_context=False, extra_props={
  "lite_interpreter": miniutils.quote(str(int(True)))}),
- IOSJob(XCODE_VERSION, ArchVariant("x86_64", "full_jit"), is_org_member_context=False, extra_props={
- "lite_interpreter": miniutils.quote(str(int(False)))}),
  IOSJob(XCODE_VERSION, ArchVariant("arm64"), extra_props={
  "lite_interpreter": miniutils.quote(str(int(True)))}),
  IOSJob(XCODE_VERSION, ArchVariant("arm64", "metal"), extra_props={
  "use_metal": miniutils.quote(str(int(True))),
  "lite_interpreter": miniutils.quote(str(int(True)))}),
- IOSJob(XCODE_VERSION, ArchVariant("arm64", "full_jit"), extra_props={
- "lite_interpreter": miniutils.quote(str(int(False)))}),
- IOSJob(XCODE_VERSION, ArchVariant("arm64", "custom"), extra_props={
+ IOSJob(XCODE_VERSION, ArchVariant("arm64", "custom-ops"), extra_props={
  "op_list": "mobilenetv2.yaml",
  "lite_interpreter": miniutils.quote(str(int(True)))}),
  IOSJob(XCODE_VERSION, ArchVariant("x86_64", "coreml"), is_org_member_context=False, extra_props={

.circleci/cimodel/data/simple/macos_definitions.py

@@ -1,3 +1,7 @@
+from collections import OrderedDict
+from cimodel.lib.miniutils import quote
+
+
 class MacOsJob:
  def __init__(self, os_version, is_build=False, is_test=False, extra_props=tuple()):
  # extra_props is tuple type, because mutable data structures for argument defaults
@@ -11,10 +15,14 @@ def gen_tree(self):
  non_phase_parts = ["pytorch", "macos", self.os_version, "py3"]
 
  extra_name_list = [name for name, exist in self.extra_props.items() if exist]
- full_job_name_list = non_phase_parts + extra_name_list + [
- 'build' if self.is_build else None,
- 'test' if self.is_test else None,
- ]
+ full_job_name_list = (
+ non_phase_parts
+ + extra_name_list
+ + [
+ "build" if self.is_build else None,
+ "test" if self.is_test else None,
+ ]
+ )
 
  full_job_name = "_".join(list(filter(None, full_job_name_list)))
 
@@ -41,12 +49,93 @@ def gen_tree(self):
  "10_13",
  is_build=True,
  is_test=True,
- extra_props=tuple({
- "lite_interpreter": True
- }.items()),
- )
+ extra_props=tuple({"lite_interpreter": True}.items()),
+ ),
 ]
 
 
+def get_new_workflow_jobs():
+ return [
+ OrderedDict(
+ {
+ "mac_build": OrderedDict(
+ {
+ "name": "macos-12-py3-x86-64-build",
+ "build-environment": "macos-12-py3-x86-64",
+ "xcode-version": quote("13.3.1"),
+ }
+ )
+ }
+ ),
+ OrderedDict(
+ {
+ "mac_test": OrderedDict(
+ {
+ "name": "macos-12-py3-x86-64-test-1-2-default",
+ "build-environment": "macos-12-py3-x86-64",
+ "xcode-version": quote("13.3.1"),
+ "shard-number": quote("1"),
+ "num-test-shards": quote("2"),
+ "requires": ["macos-12-py3-x86-64-build"],
+ }
+ )
+ }
+ ),
+ OrderedDict(
+ {
+ "mac_test": OrderedDict(
+ {
+ "name": "macos-12-py3-x86-64-test-2-2-default",
+ "build-environment": "macos-12-py3-x86-64",
+ "xcode-version": quote("13.3.1"),
+ "shard-number": quote("2"),
+ "num-test-shards": quote("2"),
+ "requires": ["macos-12-py3-x86-64-build"],
+ }
+ )
+ }
+ ),
+ OrderedDict(
+ {
+ "mac_test": OrderedDict(
+ {
+ "name": "macos-12-py3-x86-64-test-1-1-functorch",
+ "build-environment": "macos-12-py3-x86-64",
+ "xcode-version": quote("13.3.1"),
+ "shard-number": quote("1"),
+ "num-test-shards": quote("1"),
+ "test-config": "functorch",
+ "requires": ["macos-12-py3-x86-64-build"],
+ }
+ )
+ }
+ ),
+ OrderedDict(
+ {
+ "mac_build": OrderedDict(
+ {
+ "name": "macos-12-py3-x86-64-lite-interpreter-build-test",
+ "build-environment": "macos-12-py3-lite-interpreter-x86-64",
+ "xcode-version": quote("13.3.1"),
+ "build-generates-artifacts": "false",
+ }
+ )
+ }
+ ),
+ OrderedDict(
+ {
+ "mac_build": OrderedDict(
+ {
+ "name": "macos-12-py3-arm64-build",
+ "build-environment": "macos-12-py3-arm64",
+ "xcode-version": quote("13.3.1"),
+ "python-version": quote("3.9.12"),
+ }
+ )
+ }
+ ),
+ ]
+
+
 def get_workflow_jobs():
  return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/nightly_ios.py

@@ -15,7 +15,7 @@ def __init__(self,
  def get_phase_name(self):
  return "upload" if self.is_upload else "build"
 
- def get_common_name_pieces(self, with_version_dots):
+ def get_common_name_pieces(self, sep):
 
  extra_name_suffix = [self.get_phase_name()] if self.is_upload else []
 
@@ -24,7 +24,7 @@ def get_common_name_pieces(self, with_version_dots):
  common_name_pieces = [
  "ios",
  ] + extra_name + [
- ] + ios_definitions.XCODE_VERSION.render_dots_or_parts(with_version_dots) + [
+ ] + ios_definitions.XCODE_VERSION.render_dots_or_parts(sep) + [
  "nightly",
  self.variant,
  "build",
@@ -33,14 +33,14 @@ def get_common_name_pieces(self, with_version_dots):
  return common_name_pieces
 
  def gen_job_name(self):
- return "_".join(["pytorch"] + self.get_common_name_pieces(False))
+ return "_".join(["pytorch"] + self.get_common_name_pieces(None))
 
  def gen_tree(self):
  build_configs = BUILD_CONFIGS_FULL_JIT if self.is_full_jit else BUILD_CONFIGS
  extra_requires = [x.gen_job_name() for x in build_configs] if self.is_upload else []
 
  props_dict = {
- "build_environment": "-".join(["libtorch"] + self.get_common_name_pieces(True)),
+ "build_environment": "-".join(["libtorch"] + self.get_common_name_pieces(".")),
  "requires": extra_requires,
  "context": "org-member",
  "filters": {"branches": {"only": "nightly"}},

.circleci/cimodel/data/simple/upload_test_stats_definition.py

@@ -0,0 +1,20 @@
+from typing import OrderedDict
+
+
+def get_workflow_job():
+ return [
+ OrderedDict(
+ {
+ "upload_test_stats": OrderedDict(
+ {
+ "name": "upload test status",
+ "requires": [
+ "macos-12-py3-x86-64-test-1-2-default",
+ "macos-12-py3-x86-64-test-2-2-default",
+ "macos-12-py3-x86-64-test-1-1-functorch",
+ ],
+ }
+ )
+ }
+ ),
+ ]

.circleci/cimodel/data/simple/util/versions.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class MultiPartVersion:
  def __init__(self, parts, prefix=""):
  self.parts = parts
@@ -13,14 +16,11 @@ def prefixed_parts(self):
  else:
  return [self.prefix]
 
- def render_dots(self):
- return ".".join(self.prefixed_parts())
-
- def render_dots_or_parts(self, with_dots):
- if with_dots:
- return [self.render_dots()]
- else:
+ def render_dots_or_parts(self, sep: Optional[str] = None):
+ if sep is None:
  return self.prefixed_parts()
+ else:
+ return [sep.join(self.prefixed_parts())]
 
 
 class CudaVersion(MultiPartVersion):