Merge pull request #332 from populationgenomics/upstream-14301

Merge upstream HEAD(dc7fce0, 2024-05-14) Use CI's credentials for image pushing instead of gcr-push

Makefile

@@ -11,7 +11,7 @@ CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
 SPECIAL_IMAGES := hail-ubuntu batch-worker letsencrypt
 
 HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetics-$(img))
-CI_IMAGES = ci-utils ci-buildkit base hail-run
+CI_IMAGES = ci-utils hail-buildkit base hail-run
 PRIVATE_REGISTRY_IMAGES = $(patsubst %, pushed-private-%-image, $(SPECIAL_IMAGES) $(SERVICES_PLUS_ADMIN_POD) $(CI_IMAGES) $(HAILGENETICS_IMAGES))
 
 HAILTOP_VERSION := hail/python/hailtop/hail_version

build.yaml

@@ -39,7 +39,6 @@ steps:
     namespaceName: default
     secrets:
       - name: auth-oauth2-client-secret
-      - name: registry-push-credentials
       - name: hail-ci-0-1-github-oauth-token
       - name: testns-test-gsa-key
       - name: testns-test-dev-gsa-key
@@ -76,11 +75,11 @@ steps:
       REGISTRY={{ global.docker_prefix.split('/')[0] }}
 
       {% if global.cloud == "gcp" %}
-      cat /registry-push-credentials/credentials.json | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
+      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
       {% elif global.cloud == "azure" %}
       dnf install -y jq
-      USERNAME=$(cat /registry-push-credentials/credentials.json | jq -jr '.appId')
-      cat /registry-push-credentials/credentials.json | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
+      USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.appId')
+      cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
       {% else %}
       echo "unknown cloud {{ global.cloud }}"
       exit 1
@@ -91,11 +90,6 @@ steps:
     inputs:
       - from: /repo/docker
         to: /io/docker
-    secrets:
-      - name: registry-push-credentials
-        namespace:
-          valueFrom: default_ns.name
-        mountPath: /registry-push-credentials
     scopes:
       - deploy
     dependsOn:
@@ -409,6 +403,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     runIfRequested: true
     scopes:
       - dev
@@ -581,6 +576,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     secrets:
       - name:
           valueFrom: auth_database.user_secret_name
@@ -1759,6 +1755,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     runIfRequested: true
     scopes:
       - dev
@@ -1983,6 +1980,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     runIfRequested: true
     scopes:
       - dev
@@ -2011,6 +2009,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     runIfRequested: true
     scopes:
       - dev
@@ -3479,11 +3478,11 @@ steps:
 
       set +x
       {% if global.cloud == "gcp" %}
-      cat /registry-push-credentials/credentials.json | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
+      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
       {% elif global.cloud == "azure" %}
       dnf install -y jq
-      USERNAME=$(cat /registry-push-credentials/credentials.json | jq -jr '.appId')
-      cat /registry-push-credentials/credentials.json | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
+      USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.appId')
+      cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
       {% else %}
       echo "unknown cloud {{ global.cloud }}"
       exit 1
@@ -3501,11 +3500,6 @@ steps:
         to: /io/docker/hailgenetics/mirror_images.sh
       - from: /repo/docker/copy_image.sh
         to: /io/docker/copy_image.sh
-    secrets:
-      - name: registry-push-credentials
-        namespace:
-          valueFrom: default_ns.name
-        mountPath: /registry-push-credentials
     scopes:
       - deploy
       - dev

ci/ci/build.py

@@ -406,28 +406,15 @@ def build(self, batch, code, scope):
 
         docker_registry = DOCKER_PREFIX.split('/')[0]
         job_env = {'REGISTRY': docker_registry}
-        if CLOUD == 'gcp':
-            credentials_name = 'GOOGLE_APPLICATION_CREDENTIALS'
-        else:
-            assert CLOUD == 'azure'
-            credentials_name = 'AZURE_APPLICATION_CREDENTIALS'
-        credentials_secret = {
-            'namespace': DEFAULT_NAMESPACE,
-            'name': 'registry-push-credentials',
-            'mount_path': '/secrets/registry-push-credentials',
-        }
-        job_env[credentials_name] = '/secrets/registry-push-credentials/credentials.json'
 
         self.job = batch.create_job(
             BUILDKIT_IMAGE,
             command=['/bin/sh', '-c', script],
-            secrets=[credentials_secret],
             env=job_env,
             attributes={'name': self.name},
             resources=self.resources,
             input_files=input_files,
             parents=self.deps_parents(),
-            network='private',
             unconfined=True,
             regions=[REGION],
         )
@@ -445,9 +432,9 @@ def cleanup(self, batch, scope, parents):
 date
 
 set +x
-USERNAME=$(cat /secrets/registry-push-credentials/credentials.json | jq -j '.appId')
-PASSWORD=$(cat /secrets/registry-push-credentials/credentials.json | jq -j '.password')
-TENANT=$(cat /secrets/registry-push-credentials/credentials.json | jq -j '.tenant')
+USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -j '.appId')
+PASSWORD=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -j '.password')
+TENANT=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -j '.tenant')
 az login --service-principal -u $USERNAME -p $PASSWORD --tenant $TENANT
 set -x
 
@@ -467,9 +454,6 @@ def cleanup(self, batch, scope, parents):
 set -x
 date
 
-gcloud -q auth activate-service-account \
-  --key-file=/secrets/registry-push-credentials/credentials.json
-
 until gcloud -q container images untag {shq(self.image)} || ! gcloud -q container images describe {shq(self.image)}
 do
     echo 'failed, will sleep 2 and retry'
@@ -484,17 +468,9 @@ def cleanup(self, batch, scope, parents):
             image,
             command=['bash', '-c', script],
             attributes={'name': f'cleanup_{self.name}'},
-            secrets=[
-                {
-                    'namespace': DEFAULT_NAMESPACE,
-                    'name': 'registry-push-credentials',
-                    'mount_path': '/secrets/registry-push-credentials',
-                }
-            ],
             resources={'cpu': '0.25'},
             parents=parents,
             always_run=True,
-            network='private',
             timeout=5 * 60,
             regions=[REGION],
         )
@@ -515,6 +491,7 @@ def __init__(
         always_run,
         timeout,
         num_splits,
+        network,
     ):  # pylint: disable=unused-argument
         super().__init__(params)
         self.image = expand_value_from(image, self.input_config(params.code, params.scope))
@@ -536,6 +513,7 @@ def __init__(
         self.timeout = timeout
         self.jobs = []
         self.num_splits = num_splits
+        self.network = network
 
     def wrapped_job(self):
         return self.jobs
@@ -556,6 +534,7 @@ def from_json(params: StepParameters):
             json.get('alwaysRun', False),
             json.get('timeout', 3600),
             json.get('numSplits', 1),
+            json.get('network', 'public'),
         )
 
     def config(self, scope):  # pylint: disable=unused-argument
@@ -619,7 +598,7 @@ def _build_job(self, batch, code, scope, job_name, env, output_prefix):
             parents=self.deps_parents(),
             always_run=self.always_run,
             timeout=self.timeout,
-            network='private',
+            network=self.network,
             env=env,
             regions=[REGION],
         )
@@ -629,11 +608,12 @@ def cleanup(self, batch, scope, parents):
 
 
 class CreateNamespaceStep(Step):
-    def __init__(self, params, namespace_name, secrets):
+    def __init__(self, params, namespace_name, secrets, network):
         super().__init__(params)
         self.namespace_name = namespace_name
         self.secrets = secrets
         self.job = None
+        self.network = network
 
         if is_test_deployment:
             assert self.namespace_name == 'default'
@@ -661,6 +641,7 @@ def from_json(params: StepParameters):
             params,
             json['namespaceName'],
             json.get('secrets'),
+            json.get('network', 'private'),
         )
 
     def config(self, scope):  # pylint: disable=unused-argument
@@ -765,7 +746,7 @@ def build(self, batch, code, scope):  # pylint: disable=unused-argument
             # FIXME configuration
             service_account={'namespace': DEFAULT_NAMESPACE, 'name': 'ci-agent'},
             parents=self.deps_parents(),
-            network='private',
+            network=self.network,
             regions=[REGION],
         )
 
@@ -794,7 +775,7 @@ def cleanup(self, batch, scope, parents):
             service_account={'namespace': DEFAULT_NAMESPACE, 'name': 'ci-agent'},
             parents=parents,
             always_run=True,
-            network='private',
+            network=self.network,
             regions=[REGION],
         )
 
@@ -927,7 +908,6 @@ def build(self, batch, code, scope):
             service_account={'namespace': DEFAULT_NAMESPACE, 'name': 'ci-agent'},
             resources={'cpu': '0.25'},
             parents=self.deps_parents(),
-            network='private',
             regions=[REGION],
         )
 
@@ -954,7 +934,6 @@ def cleanup(self, batch, scope, parents):  # pylint: disable=unused-argument
                 resources={'cpu': '0.25'},
                 parents=parents,
                 always_run=True,
-                network='private',
                 regions=[REGION],
             )
 

ci/test/resources/build.yaml

@@ -290,6 +290,7 @@ steps:
       name: admin
       namespace:
         valueFrom: default_ns.name
+    network: private
     dependsOn:
       - default_ns
       - hello_database

gear/pinned-requirements.txt

@@ -98,7 +98,7 @@ multidict==6.0.4
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   aiohttp
     #   yarl
-orjson==3.9.12
+orjson==3.9.11
     # via
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
     #   -c hail/gear/../hail/python/pinned-requirements.txt

gear/requirements.txt

@@ -14,4 +14,5 @@ prometheus_async>=19.2.0,<20
 prometheus_client>=0.11.0,<1
 PyMySQL>=1,<2
 sortedcontainers>=2.4.0,<3
-orjson>=3.6.4,<4
+# <3.9.12: https://github.com/hail-is/hail/issues/14299
+orjson>=3.6.4,<3.9.12

hail/build.sc

@@ -12,7 +12,7 @@ import mill.util.Jvm
 
 object Settings {
   val hailMajorMinorVersion = "0.2"
-  val hailPatchVersion = "127"
+  val hailPatchVersion = "128"
 }
 
 /** Update the millw script. */
@@ -197,13 +197,11 @@ object main extends RootModule with HailScalaModule { outer =>
     Deps.Breeze.natives.excludeOrg("org.apache.commons.math3"),
     Deps.Commons.io,
     Deps.Commons.lang3,
-    //    ivy"org.apache.commons:commons-math3:3.6.1",
     Deps.Commons.codec,
     Deps.lz4,
     Deps.netlib,
     Deps.avro.excludeOrg("com.fasterxml.jackson.core"),
     Deps.junixsocket,
-//    Deps.zstd
   )
 
   override def compileIvyDeps: T[Agg[Dep]] = Agg(
@@ -212,7 +210,6 @@ object main extends RootModule with HailScalaModule { outer =>
     Deps.Spark.core(),
     Deps.Spark.mllib(),
     Deps.Breeze.core,
-    //      ivy"org.scalanlp::breeze-natives:1.1",
   )
 
   override def assemblyRules: Seq[Rule] = super.assemblyRules ++ Seq(
@@ -257,19 +254,13 @@ object main extends RootModule with HailScalaModule { outer =>
     override def sources: T[Seq[PathRef]] = T.sources {
       Seq(PathRef(this.millSourcePath / os.up / "src" / debugOrRelease() / "java"))
     }
-
-    override def compileIvyDeps: T[Agg[Dep]] = Agg(
-      Deps.hadoopClient,
-      Deps.samtools.excludeOrg("*"),
-    )
   }
 
   object test extends HailTests {
     override def resources: T[Seq[PathRef]] = outer.resources() ++ super.resources()
 
     override def assemblyRules: Seq[Rule] = outer.assemblyRules ++ Seq(
       Rule.Relocate("org.codehaus.jackson.**", "is.hail.relocated.@0")
-//      Rule.Relocate("org.codehaus.stax2.**", "is.hail.relocated.@0"),
     )
 
     override def ivyDeps: T[Agg[Dep]] = super.ivyDeps() ++ Seq(

hail/python/hail/docs/change_log.md

@@ -55,7 +55,7 @@ critically depend on experimental functionality.**
 
 ## Version 0.2.128
 
-Released 2024-02-14
+Released 2024-02-15
 
 In GCP, the Hail Annotation DB and Datasets API have moved from multi-regional US and EU buckets to
 regional US-CENTRAL1 and EUROPE-WEST1 buckets. These buckets are requester pays which means unless
@@ -84,6 +84,7 @@ require us to choose only one region per continent and we have chosen US-CENTRAL
 
 ### Bug Fixes
 
+- (hail#14300) Require orjson<3.9.12 to avoid a segfault introduced in orjson 3.9.12
 - (hail#14071) Use indexed VEP cache files for GRCh38 on both dataproc and QoB.
 - (hail#14232) Allow use of large numbers of fields on a table without triggering
   `ClassTooLargeException: Class too large:`.

hail/python/hailtop/pinned-requirements.txt

@@ -106,7 +106,7 @@ nest-asyncio==1.6.0
     # via -r hail/hail/python/hailtop/requirements.txt
 oauthlib==3.2.2
     # via requests-oauthlib
-orjson==3.9.12
+orjson==3.9.11
     # via -r hail/hail/python/hailtop/requirements.txt
 packaging==23.2
     # via msal-extensions

hail/python/hailtop/requirements.txt

@@ -12,7 +12,8 @@ google-auth-oauthlib>=0.5.2,<1
 humanize>=1.0.0,<2
 janus>=0.6,<1.1
 nest_asyncio>=1.5.8,<2
-orjson>=3.6.4,<4
+# <3.9.12: https://github.com/hail-is/hail/issues/14299
+orjson>=3.6.4,<3.9.12
 protobuf==3.20.2
 rich>=12.6.0,<13
 typer>=0.9.0,<1

hail/python/pinned-requirements.txt

@@ -187,7 +187,7 @@ oauthlib==3.2.2
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   requests-oauthlib
-orjson==3.9.12
+orjson==3.9.11
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt

hail/version.mk

@@ -16,7 +16,7 @@ endif
 SCALA_VERSION ?= 2.12.18
 SPARK_VERSION ?= 3.3.2
 HAIL_MAJOR_MINOR_VERSION := 0.2
-HAIL_PATCH_VERSION := 127
+HAIL_PATCH_VERSION := 128
 HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION)
 HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION)
 ELASTIC_MAJOR_VERSION ?= 7