Skip to content

Commit 93a45d0

Browse files
def-def-
authored
Merge pull request #34092 from def-/pr-ghcr-upgrade
tests: Use GHCR for older Mz versions too
2 parents 237942b + 62018a4 commit 93a45d0

File tree

13 files changed

+75
-60
lines changed

13 files changed

+75
-60
lines changed

‎misc/python/materialize/checks/mzcompose_actions.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from materialize import MZ_ROOT
1515
from materialize.checks.actions import Action
1616
from materialize.checks.executors import Executor
17+
from materialize.docker import image_registry
1718
from materialize.mz_version import MzVersion
1819
from materialize.mzcompose.services.clusterd import Clusterd
1920
from materialize.mzcompose.services.materialized import DeploymentStatus, Materialized
@@ -67,7 +68,11 @@ def __init__(
6768
def execute(self, e: Executor) -> None:
6869
c = e.mzcompose_composition()
6970

70-
image = f"materialize/materialized:{self.tag}" if self.tag is not None else None
71+
image = (
72+
f"{image_registry()}/materialized:{self.tag}"
73+
if self.tag is not None
74+
else None
75+
)
7176
print(f"Starting Mz using image {image}, mz_service {self.mz_service}")
7277

7378
listeners_config_path = (

‎misc/python/materialize/cli/ci_upload_heap_profiles.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,18 @@ def run(service: str, backend: list[str], suffix: str = ""):
6969
).stdout
7070
)
7171
for service in services:
72-
if service["Image"].startswith("materialize/clusterd:"):
72+
image = service["Image"].rsplit(":", 1)[0]
73+
ghcr_prefix = "ghcr.io/materializeinc/"
74+
if image.startswith(ghcr_prefix):
75+
image.removeprefix(ghcr_prefix)
76+
if image == "materialize/clusterd":
7377
threads.append(
7478
Thread(
7579
target=run,
7680
args=(service["Service"], ["http://127.0.0.1:6878/heap"]),
7781
)
7882
)
79-
elif service["Image"].startswith("materialize/materialized:"):
83+
elif image == "materialize/materialized":
8084
threads.append(
8185
Thread(
8286
target=run,

‎misc/python/materialize/cli/orchestratord.py‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,14 @@
2424
import yaml
2525

2626
from materialize import MZ_ROOT, ui
27-
from materialize.docker import MZ_GHCR_DEFAULT
27+
from materialize.docker import image_registry
2828

2929
DEV_IMAGE_TAG = "local-dev"
3030
DEFAULT_POSTGRES = (
3131
"postgres://root@postgres.materialize.svc.cluster.local:5432/materialize"
3232
)
3333
DEFAULT_MINIO = "s3://minio:minio123@persist/persist?endpoint=http%3A%2F%2Fminio.materialize.svc.cluster.local%3A9000&region=minio"
3434

35-
IMAGE_REGISTRY = (
36-
"ghcr.io/materializeinc/materialize"
37-
if ui.env_is_truthy("MZ_GHCR", MZ_GHCR_DEFAULT)
38-
else "materialize"
39-
)
40-
4135

4236
def main():
4337
os.chdir(MZ_ROOT)
@@ -120,7 +114,7 @@ def run(args: argparse.Namespace):
120114
"misc/helm-charts/operator",
121115
"--atomic",
122116
f"--set=operator.image.tag={DEV_IMAGE_TAG}",
123-
f"--set=operator.image.repository={IMAGE_REGISTRY}/orchestratord",
117+
f"--set=operator.image.repository={image_registry()}/orchestratord",
124118
"--create-namespace",
125119
f"--namespace={args.namespace}",
126120
]
@@ -198,7 +192,7 @@ def environment(args: argparse.Namespace):
198192
cluster=args.kind_cluster_name,
199193
)
200194
image_tag = DEV_IMAGE_TAG
201-
environmentd_image_ref = f"{IMAGE_REGISTRY}/environmentd:{image_tag}"
195+
environmentd_image_ref = f"{image_registry()}/environmentd:{image_tag}"
202196

203197
try:
204198
kubectl(
@@ -445,14 +439,14 @@ def acquire(image: str, dev: bool, cluster: str):
445439
[
446440
"docker",
447441
"tag",
448-
f"{IMAGE_REGISTRY}/{image}:mzbuild-{fingerprint}",
449-
f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}",
442+
f"{image_registry()}/{image}:mzbuild-{fingerprint}",
443+
f"{image_registry()}/{image}:{DEV_IMAGE_TAG}",
450444
]
451445
)
452446
kind(
453447
"load",
454448
"docker-image",
455-
f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}",
449+
f"{image_registry()}/{image}:{DEV_IMAGE_TAG}",
456450
cluster=cluster,
457451
)
458452

‎misc/python/materialize/docker.py‎

Lines changed: 34 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,29 @@ def image_of_commit_exists(commit_hash: str) -> bool:
4343
return mz_image_tag_exists(commit_to_image_tag(commit_hash))
4444

4545

46+
def mz_image_tag_exists_cmdline(image_name: str) -> bool:
47+
command = [
48+
"docker",
49+
"manifest",
50+
"inspect",
51+
image_name,
52+
]
53+
try:
54+
subprocess.check_output(command, stderr=subprocess.STDOUT, text=True)
55+
EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
56+
return True
57+
except subprocess.CalledProcessError as e:
58+
if "no such manifest:" in e.output:
59+
print(f"Failed to fetch image manifest '{image_name}' (does not exist)")
60+
EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = False
61+
else:
62+
print(f"Failed to fetch image manifest '{image_name}' ({e.output})")
63+
# do not cache the result of unknown error messages
64+
return False
65+
66+
4667
def mz_image_tag_exists(image_tag: str) -> bool:
47-
image_name = f"materialize/materialized:{image_tag}"
68+
image_name = f"{image_registry()}/materialized:{image_tag}"
4869

4970
if image_name in EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK:
5071
image_exists = EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name]
@@ -67,30 +88,16 @@ def mz_image_tag_exists(image_tag: str) -> bool:
6788
# when the image doesn't exist, see https://www.docker.com/increase-rate-limits/,
6889
# so use the API instead.
6990

91+
if image_registry() != "materialize":
92+
return mz_image_tag_exists_cmdline(image_name)
93+
7094
try:
7195
response = requests.get(
7296
f"https://hub.docker.com/v2/repositories/materialize/materialized/tags/{image_tag}"
7397
)
7498
result = response.json()
7599
except (requests.exceptions.ConnectionError, requests.exceptions.JSONDecodeError):
76-
command = [
77-
"docker",
78-
"manifest",
79-
"inspect",
80-
image_name,
81-
]
82-
try:
83-
subprocess.check_output(command, stderr=subprocess.STDOUT, text=True)
84-
EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
85-
return True
86-
except subprocess.CalledProcessError as e:
87-
if "no such manifest:" in e.output:
88-
print(f"Failed to fetch image manifest '{image_name}' (does not exist)")
89-
EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = False
90-
else:
91-
print(f"Failed to fetch image manifest '{image_name}' ({e.output})")
92-
# do not cache the result of unknown error messages
93-
return False
100+
return mz_image_tag_exists_cmdline(image_name)
94101

95102
if result.get("images"):
96103
EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
@@ -198,3 +205,11 @@ def _select_image_name_from_candidates(
198205
)
199206

200207
return image_name_candidates[0]
208+
209+
210+
def image_registry() -> str:
211+
return (
212+
"ghcr.io/materializeinc/materialize"
213+
if ui.env_is_truthy("MZ_GHCR", "1")
214+
else "materialize"
215+
)

‎misc/python/materialize/mzbuild.py‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
from requests.auth import HTTPBasicAuth
4747

4848
from materialize import MZ_ROOT, buildkite, cargo, git, rustc_flags, spawn, ui, xcompile
49-
from materialize.docker import MZ_GHCR_DEFAULT
49+
from materialize.docker import image_registry
5050
from materialize.rustc_flags import Sanitizer
5151
from materialize.xcompile import Arch, target
5252

@@ -1263,11 +1263,7 @@ def __init__(
12631263
),
12641264
coverage: bool = False,
12651265
sanitizer: Sanitizer = Sanitizer.none,
1266-
image_registry: str = (
1267-
"ghcr.io/materializeinc/materialize"
1268-
if ui.env_is_truthy("MZ_GHCR", MZ_GHCR_DEFAULT)
1269-
else "materialize"
1270-
),
1266+
image_registry: str = image_registry(),
12711267
image_prefix: str = "",
12721268
):
12731269
self.rd = RepositoryDetails(
@@ -1369,12 +1365,7 @@ def install_arguments(parser: argparse.ArgumentParser) -> None:
13691365
)
13701366
parser.add_argument(
13711367
"--image-registry",
1372-
default=(
1373-
"ghcr.io/materializeinc/materialize"
1374-
if ui.env_is_truthy("CI")
1375-
or ui.env_is_truthy("MZ_GHCR", MZ_GHCR_DEFAULT)
1376-
else "materialize"
1377-
),
1368+
default=image_registry(),
13781369
help="the Docker image registry to pull images from and push images to",
13791370
)
13801371
parser.add_argument(

‎misc/python/materialize/mzcompose/composition.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
from psycopg import Connection, Cursor
4646

4747
from materialize import MZ_ROOT, buildkite, mzbuild, spawn, ui
48+
from materialize.docker import image_registry
4849
from materialize.mzcompose import cluster_replica_size_map, loader
4950
from materialize.mzcompose.service import Service as MzComposeService
5051
from materialize.mzcompose.services.materialized import (
@@ -1124,7 +1125,7 @@ def final_preflight_check(self) -> None:
11241125
self.kill("materialized")
11251126
with self.override(
11261127
Materialized(
1127-
image=f"materialize/materialized:{version}",
1128+
image=f"{image_registry()}/materialized:{version}",
11281129
environment_extra=["MZ_DEPLOY_GENERATION=1"],
11291130
healthcheck=LEADER_STATUS_HEALTHCHECK,
11301131
)

‎test/feature-benchmark/mzcompose.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from textwrap import dedent
2222

2323
from materialize import buildkite
24-
from materialize.docker import is_image_tag_of_release_version
24+
from materialize.docker import image_registry, is_image_tag_of_release_version
2525
from materialize.feature_benchmark.benchmark_result_evaluator import (
2626
BenchmarkResultEvaluator,
2727
)
@@ -211,7 +211,7 @@ def run_one_scenario(
211211
param_name, param_value = param.split("=")
212212
additional_system_parameter_defaults[param_name] = param_value
213213

214-
mz_image = f"materialize/materialized:{tag}" if tag else None
214+
mz_image = f"{image_registry()}/materialized:{tag}" if tag else None
215215
# TODO: Better azurite support detection
216216
mz = create_mz_service(
217217
mz_image,
@@ -228,7 +228,7 @@ def run_one_scenario(
228228
print(
229229
f"Unable to find materialize image with tag {tag}, proceeding with latest instead!"
230230
)
231-
mz_image = "materialize/materialized:latest"
231+
mz_image = "{image_registry()}/materialized:latest"
232232
# TODO: Better azurite support detection
233233
mz = create_mz_service(
234234
mz_image,

‎test/legacy-upgrade/mzcompose.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import random
1616

1717
from materialize import buildkite
18-
from materialize.docker import image_of_release_version_exists
18+
from materialize.docker import image_of_release_version_exists, image_registry
1919
from materialize.mz_version import MzVersion
2020
from materialize.mzcompose import get_default_system_parameters
2121
from materialize.mzcompose.composition import Composition, WorkflowArgumentParser
@@ -201,7 +201,7 @@ def test_upgrade_from_version(
201201
)
202202
mz_from = Materialized(
203203
name=mz_service,
204-
image=f"materialize/materialized:{from_version}",
204+
image=f"{image_registry()}/materialized:{from_version}",
205205
options=[
206206
opt
207207
for start_version, opt in mz_options.items()
@@ -273,7 +273,7 @@ def test_upgrade_from_version(
273273
with c.override(
274274
Materialized(
275275
name=mz_service,
276-
image=f"materialize/materialized:{version}",
276+
image=f"{image_registry()}/materialized:{version}",
277277
options=[
278278
opt
279279
for start_version, opt in mz_options.items()

‎test/mzcompose_examples/mzcompose.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
# the Business Source License, use of this software will be governed
88
# by the Apache License, Version 2.0.
99

10+
from materialize.docker import image_registry
1011
from materialize.mzcompose.composition import Composition
1112
from materialize.mzcompose.services.kafka import Kafka
1213
from materialize.mzcompose.services.materialized import Materialized
@@ -19,7 +20,7 @@
1920
versioned_mz = [
2021
Materialized(
2122
name=f"materialized_{version}",
22-
image=f"materialize/materialized:{version}",
23+
image=f"{image_registry()}/materialized:{version}",
2324
)
2425
for version in ["v0.7.0", "v0.8.0"]
2526
]

‎test/parallel-benchmark/mzcompose.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from matplotlib.markers import MarkerStyle
2323

2424
from materialize import MZ_ROOT, buildkite
25+
from materialize.docker import image_registry
2526
from materialize.mz_env_util import get_cloud_hostname
2627
from materialize.mzcompose import ADDITIONAL_BENCHMARKING_SYSTEM_PARAMETERS
2728
from materialize.mzcompose.composition import (
@@ -440,7 +441,7 @@ def run_once(
440441
else:
441442
overrides = [
442443
Materialized(
443-
image=f"materialize/materialized:{tag}" if tag else None,
444+
image=f"{image_registry()}/materialized:{tag}" if tag else None,
444445
default_size=args.size,
445446
soft_assertions=False,
446447
external_metadata_store=True,

0 commit comments

Comments
 (0)