Merge pull request #344 from populationgenomics/upstream-0.2.132

Merge upstream 0.2.132 release (678e1f5, 2024-07-09).

.dockerignore

@@ -17,6 +17,7 @@ hail/build/
 hail/hail.iml
 hail/hail.log
 hail/out/
+!hail/out/assembly.dest/out.jar
 hail/target/
 hail/python/hail/docs/_build/*
 hail/src/main/c/headers
@@ -27,4 +28,5 @@ hail/src/main/resources/build-info.properties
 hail/src/test/resources/example.v11.bgen.idx
 hail/src/test/resources/random.bgen.idx
 benchmark/python/build/
+web_common/web_common/static/css
 ci/repos

Makefile

@@ -89,61 +89,25 @@ check-pip-requirements:
 		batch \
 		ci
 
-.PHONY: check-linux-pip-requirements
-check-linux-pip-requirements:
-	./check_linux_pip_requirements.sh \
-		hail/python/hailtop \
-		hail/python \
-		hail/python/dev \
-		gear \
-		web_common \
-		batch \
-		ci
-
 .PHONY: install-dev-requirements
 install-dev-requirements:
 	python3 -m pip install \
 		-r hail/python/pinned-requirements.txt \
 		-r hail/python/dev/pinned-requirements.txt \
-		-r benchmark/python/pinned-requirements.txt \
 		-r gear/pinned-requirements.txt \
 		-r web_common/pinned-requirements.txt \
 		-r batch/pinned-requirements.txt \
 		-r ci/pinned-requirements.txt
 
-hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
-	./generate-linux-pip-lockfile.sh hail/python/hailtop
-
-hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt hail/python/requirements.txt
-	./generate-linux-pip-lockfile.sh hail/python
-
-hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
-	./generate-linux-pip-lockfile.sh hail/python/dev
-
-benchmark/python/pinned-requirements.txt: benchmark/python/requirements.txt hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt
-	./generate-linux-pip-lockfile.sh benchmark/python
-
-gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
-	./generate-linux-pip-lockfile.sh gear
-
-web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt
-	./generate-linux-pip-lockfile.sh web_common
-
-batch/pinned-requirements.txt: web_common/pinned-requirements.txt batch/requirements.txt
-	./generate-linux-pip-lockfile.sh batch
-
-ci/pinned-requirements.txt: web_common/pinned-requirements.txt ci/requirements.txt
-	./generate-linux-pip-lockfile.sh ci
-
 .PHONY: generate-pip-lockfiles
-generate-pip-lockfiles: hail/python/hailtop/pinned-requirements.txt
-generate-pip-lockfiles: hail/python/pinned-requirements.txt
-generate-pip-lockfiles: hail/python/dev/pinned-requirements.txt
-generate-pip-lockfiles: benchmark/python/pinned-requirements.txt
-generate-pip-lockfiles: gear/pinned-requirements.txt
-generate-pip-lockfiles: web_common/pinned-requirements.txt
-generate-pip-lockfiles: batch/pinned-requirements.txt
-generate-pip-lockfiles: ci/pinned-requirements.txt
+generate-pip-lockfiles:
+	./generate-pip-lockfile.sh hail/python/hailtop
+	./generate-pip-lockfile.sh hail/python
+	./generate-pip-lockfile.sh hail/python/dev
+	./generate-pip-lockfile.sh gear
+	./generate-pip-lockfile.sh web_common
+	./generate-pip-lockfile.sh batch
+	./generate-pip-lockfile.sh ci
 
 $(HAILTOP_VERSION):
 	$(MAKE) -C hail python/hailtop/hail_version
@@ -166,11 +130,16 @@ hail-run-image: base-image hail/Dockerfile.hail-run hail/python/pinned-requireme
 	echo $(IMAGE_NAME) > $@
 
 hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
-	$(MAKE) -C hail wheel
+	$(MAKE) HAIL_RELEASE_MODE=1 -C hail wheel
 	./docker-build.sh . docker/hailgenetics/hail/Dockerfile $(IMAGE_NAME) \
 		--build-arg BASE_IMAGE=$(shell cat hail-ubuntu-image)
 	echo $(IMAGE_NAME) > $@
 
+hail-dev-image: hailgenetics-hail-image hail/python/dev/pinned-requirements.txt
+	./docker-build.sh . docker/Dockerfile.hail-dev $(IMAGE_NAME) \
+		--build-arg HAIL_IMAGE=$(shell cat $<)
+	echo $(IMAGE_NAME) > $@
+
 hail-0.1-docs-5a6778710097.tar.gz:
 	gcloud storage cp gs://hail-common/builds/0.1/docs/$@ .
 
@@ -190,7 +159,8 @@ docs.tar.gz: hail/build/www
 
 website-image: docs.tar.gz
 
-$(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $(shell git ls-files $$* ':!:**/deployment.yaml')
+.SECONDEXPANSION:
+$(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $$(shell git ls-files $$* ':!:**/deployment.yaml')
 	./docker-build.sh . $*/Dockerfile $(IMAGE_NAME) --build-arg BASE_IMAGE=$(shell cat hail-ubuntu-image)
 	echo $(IMAGE_NAME) > $@
 
@@ -220,6 +190,17 @@ hailgenetics-vep-grch38-95-image: hail-ubuntu-image
 		--build-arg BASE_IMAGE=$(shell cat hail-ubuntu-image)
 	echo $(IMAGE_NAME) > $@
 
+terra-batch-image: batch-image
+	./docker-build.sh . batch/terra-chart/Dockerfile.batch $(IMAGE_NAME) \
+		--build-arg BASE_IMAGE=$(shell cat batch-image)
+	echo $(IMAGE_NAME) > $@
+
+terra-batch-worker-image: batch-worker-image
+	$(MAKE) -C hail shadowJar HAIL_RELEASE_MODE=1
+	./docker-build.sh . batch/terra-chart/Dockerfile.worker $(IMAGE_NAME) \
+		--build-arg BASE_IMAGE=$(shell cat batch-worker-image)
+	echo $(IMAGE_NAME) > $@
+
 letsencrypt-image:
 	./docker-build.sh letsencrypt Dockerfile $(IMAGE_NAME)
 	echo $(IMAGE_NAME) > $@
@@ -246,14 +227,18 @@ $(SERVICES_DATABASES): %-db:
 	python3 ci/create_local_database.py $* local-$*
 endif
 
-.PHONY: sass-compile-watch
-sass-compile-watch:
-	cd web_common/web_common && sass --watch -I styles --style=compressed styles:static/css
+.PHONY: tailwind-compile-watch
+tailwind-compile-watch:
+	cd web_common && npx tailwindcss --watch -i input.css -o web_common/static/css/output.css
 
 .PHONY: run-dev-proxy
 run-dev-proxy:
 	SERVICE=$(SERVICE) adev runserver --root . devbin/dev_proxy.py
 
 .PHONY: devserver
 devserver:
-	$(MAKE) -j 2 sass-compile-watch run-dev-proxy
+	$(MAKE) -j 2 tailwind-compile-watch run-dev-proxy
+
+.PHONY: benchmark
+benchmark: hail-dev-image
+	$(MAKE) -C hail HAIL_DEV_IMAGE="$(shell cat $<)" benchmark

batch/Dockerfile

@@ -1,6 +1,10 @@
 ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
 FROM $BASE_IMAGE
 
+RUN curl -sLO https://github.com/tailwindlabs/tailwindcss/releases/download/v3.4.3/tailwindcss-linux-x64 && \
+    chmod +x tailwindcss-linux-x64 && \
+    mv tailwindcss-linux-x64 /usr/bin/tailwindcss
+
 COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
 COPY gear/pinned-requirements.txt gear-requirements.txt
 COPY web_common/pinned-requirements.txt web_common-requirements.txt
@@ -18,12 +22,16 @@ COPY hail/python/hailtop /hailtop/hailtop/
 COPY gear/setup.py /gear/setup.py
 COPY gear/gear /gear/gear/
 
-COPY web_common/setup.py web_common/MANIFEST.in /web_common/
+COPY web_common/setup.py web_common/MANIFEST.in web_common/input.css web_common/tailwind.config.js /web_common/
 COPY web_common/web_common /web_common/web_common/
 
 COPY batch/setup.py /batch/MANIFEST.in /batch/
 COPY batch/batch /batch/batch/
 
+RUN cd web_common && \
+    mkdir web_common/static/css && \
+    tailwindcss -i input.css -o web_common/static/css/output.css
+
 RUN hail-pip-install /hailtop /gear /web_common /batch
 
 EXPOSE 5000

batch/Dockerfile.worker

@@ -6,7 +6,7 @@ RUN hail-apt-get-install \
     iproute2 \
     iptables \
     ca-certificates-java \
-    openjdk-8-jre-headless \
+    openjdk-11-jre-headless \
     liblapack3 \
     xfsprogs \
     libyajl-dev
@@ -51,7 +51,7 @@ RUN hail-pip-install \
     -r hailtop-requirements.txt \
     -r gear-requirements.txt \
     -r batch-requirements.txt \
-    pyspark==3.3.2
+    pyspark==3.5.0
 
 ENV SPARK_HOME /usr/local/lib/python3.9/dist-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"

batch/batch/cloud/azure/driver/create_instance.py

@@ -13,7 +13,7 @@
 from ....instance_config import InstanceConfig
 from ...resource_utils import unreserved_worker_data_disk_size_gib
 from ...utils import ACCEPTABLE_QUERY_JAR_URL_PREFIX
-from ..resource_utils import azure_machine_type_to_worker_type_and_cores
+from ..resource_utils import azure_machine_type_to_parts
 
 log = logging.getLogger('create_instance')
 
@@ -43,7 +43,9 @@ def create_vm_config(
     instance_config: InstanceConfig,
     feature_flags: dict,
 ) -> dict:
-    _, cores = azure_machine_type_to_worker_type_and_cores(machine_type)
+    parts = azure_machine_type_to_parts(machine_type)
+    assert parts
+    cores = parts.cores
 
     hail_azure_oauth_scope = os.environ['HAIL_AZURE_OAUTH_SCOPE']
     region = instance_config.region_for(location)

batch/batch/cloud/azure/driver/resource_manager.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 import aiohttp
 
@@ -21,8 +21,7 @@
 from ..instance_config import AzureSlimInstanceConfig
 from ..resource_utils import (
     azure_local_ssd_size,
-    azure_machine_type_to_worker_type_and_cores,
-    azure_worker_memory_per_core_mib,
+    azure_machine_type_to_parts,
     azure_worker_properties_to_machine_type,
 )
 from .billing_manager import AzureBillingManager
@@ -101,9 +100,6 @@ async def get_vm_state(self, instance: Instance) -> VMState:
     def machine_type(self, cores: int, worker_type: str, local_ssd: bool) -> str:
         return azure_worker_properties_to_machine_type(worker_type, cores, local_ssd)
 
-    def worker_type_and_cores(self, machine_type: str) -> Tuple[str, int]:
-        return azure_machine_type_to_worker_type_and_cores(machine_type)
-
     def instance_config(
         self,
         machine_type: str,
@@ -140,10 +136,12 @@ async def create_vm(
         machine_type: str,
         instance_config: InstanceConfig,
     ) -> List[QuantifiedResource]:
-        worker_type, cores = self.worker_type_and_cores(machine_type)
+        parts = azure_machine_type_to_parts(machine_type)
+        assert parts
+        cores = parts.cores
 
         if local_ssd_data_disk:
-            assert data_disk_size_gb == azure_local_ssd_size(worker_type, cores)
+            assert data_disk_size_gb == azure_local_ssd_size(parts.family, cores)
 
         max_price: Optional[float]
         if preemptible:
@@ -173,8 +171,7 @@ async def create_vm(
             self.app['feature_flags'],
         )
 
-        memory_mib = azure_worker_memory_per_core_mib(worker_type) * cores
-        memory_in_bytes = memory_mib << 20
+        memory_in_bytes = parts.memory
         cores_mcpu = cores * 1000
         total_resources_on_instance = instance_config.quantified_resources(
             cpu_in_mcpu=cores_mcpu, memory_in_bytes=memory_in_bytes, extra_storage_in_gib=0

batch/batch/cloud/azure/instance_config.py

@@ -4,7 +4,7 @@
 
 from ...driver.billing_manager import ProductVersions
 from ...instance_config import InstanceConfig
-from .resource_utils import azure_machine_type_to_worker_type_and_cores
+from .resource_utils import azure_machine_type_to_parts
 from .resources import (
     AzureDynamicSizedDiskResource,
     AzureIPFeeResource,
@@ -73,14 +73,18 @@ def __init__(
         self.boot_disk_size_gb = boot_disk_size_gb
         self.resources = resources
 
-        worker_type, cores = azure_machine_type_to_worker_type_and_cores(self._machine_type)
-
-        self._worker_type = worker_type
-        self.cores = cores
+        machine_type_parts = azure_machine_type_to_parts(self._machine_type)
+        assert machine_type_parts
+        self.machine_type_parts = machine_type_parts
+        self._worker_type = machine_type_parts.family
+        self.cores = machine_type_parts.cores
 
     def worker_type(self) -> str:
         return self._worker_type
 
+    def instance_memory(self) -> int:
+        return self.machine_type_parts.memory
+
     def region_for(self, location: str) -> str:
         return location