Zombienet ci update

.github/workflows/zombie-builder.yml

@@ -6,8 +6,8 @@
 
 # Update zombie builder image each Sunday night
 on:
-  schedule:
-    - cron: '0 2 * * 0'
+#  schedule:
+#    - cron: '0 2 * * 0'
   workflow_dispatch:
 
 jobs:

zombienet/docker/Makefile

@@ -1,23 +1,45 @@
 builder_last:
-	python version.py https://github.com/paritytech/polkadot.git && \
-	python version.py https://github.com/paritytech/cumulus.git && \
+	python version.py https://github.com/paritytech/polkadot-sdk && \
+	python version.py https://github.com/paritytech/zombienet && \
 	make fetch_and_run
 
 fetch_and_run:
-	export $(shell [ ! -n "polkadot-version.txt" ] || cat polkadot-version.txt | grep -v \
-    --perl-regexp '^('$$(env | sed 's/=.*//'g | tr '\n' '|')')\=') && \
-	export $(shell [ ! -n "cumulus-version.txt" ] || cat cumulus-version.txt | grep -v \
-    --perl-regexp '^('$$(env | sed 's/=.*//'g | tr '\n' '|')')\=') && \
-	DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=$$CUMULUS_RELEASE --build-arg POLKADOT_RELEASE=$$POLKADOT_RELEASE --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE .
+	if [ -f polkadot-sdk-versions.txt ]; then \
+		POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
+		POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
+		POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
+		ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \
+		DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE -t qdrvm/zombie-builder:latest \
+		-f builder.Dockerfile \
+		--build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \
+		--build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \
+		--build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \
+		--build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \
+	else \
+		echo "One or more files are missing."; \
+	fi
 
 builder_push:
-	docker push qdrvm/zombie-builder:latest
+	if [ -f polkadot-sdk-versions.txt ]; then \
+  		POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
+		docker push qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE ; \
+		docker push qdrvm/zombie-builder:latest ; \
+	else \
+		echo "One or more files are missing."; \
+	fi	
 
 builder:
 	DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=v0.9.420 --build-arg POLKADOT_RELEASE=v0.9.42 --build-arg POLKADOT_RELEASE_GLOBAL=v0.9.42 .
 
 tester:
-	docker build --no-cache -t qdrvm/zombie-tester:latest -f tester.Dockerfile .
+	python version.py https://github.com/paritytech/polkadot-sdk && \
+	if [ -f polkadot-sdk-versions.txt ]; then \
+		POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
+		DOCKER_BUILDKIT=0 docker build --no-cache -t qdrvm/zombie-tester:latest -f tester.Dockerfile \
+		--build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \
+	else \
+		echo "One or more files are missing."; \
+	fi
 
 tests:
 	docker run -it qdrvm/zombie-tester:latest

zombienet/docker/builder.Dockerfile

@@ -1,17 +1,17 @@
-# Polkadot docker image to fetch
 ARG POLKADOT_RELEASE_GLOBAL
+ARG POLKADOT_RELEASE_GLOBAL_NUMERIC
+ARG ZOMBIENET_RELEASE
 
-# Base image with all dependencies
 FROM node:18-bullseye-slim as zombie-builder
 USER root
 
 RUN apt-get update && apt update && \
-    apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev && \
-    apt install -y --no-install-recommends clang ca-certificates openssl && \ 
+    apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev time && \
+    apt install -y --no-install-recommends clang ca-certificates openssl && \
     rm -rf /var/lib/apt/lists/* && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/root/.cargo/bin:$PATH
-RUN rustup default stable && rustup target add wasm32-unknown-unknown
+RUN rustup default stable && rustup target add wasm32-unknown-unknown && rustup component add rust-src
 RUN groupadd --gid 10001 nonroot && \
     useradd  --home-dir /home/nonroot \
              --create-home \
@@ -21,72 +21,50 @@ RUN groupadd --gid 10001 nonroot && \
              --uid 10000 nonroot
 WORKDIR /home/nonroot/
 
-# Image with polkadot-parachain binary
-FROM zombie-builder AS polkadot-parachain
-# Cumulus git tag to fetch
-ARG CUMULUS_RELEASE
-RUN git clone --depth 1 --branch $CUMULUS_RELEASE https://github.com/paritytech/cumulus.git
-WORKDIR /home/nonroot/cumulus
-RUN cargo build --release --bin polkadot-parachain
-
-# Image with polkadot-parachain binary
-FROM zombie-builder AS test-parachain
-# Cumulus git tag to fetch
-ARG CUMULUS_RELEASE
-RUN git clone --depth 1 --branch $CUMULUS_RELEASE https://github.com/paritytech/cumulus.git
-WORKDIR /home/nonroot/cumulus
-RUN cargo build --release --locked --bin test-parachain
-
-# Image with test-parachain-adder-collator binary
-FROM zombie-builder AS test-parachain-adder-collator
-# Polkadot git tag to fetch
-ARG POLKADOT_RELEASE
-RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git
-WORKDIR /home/nonroot/polkadot
-RUN cargo build -p test-parachain-adder-collator
-
-# Image with test-parachain-undying-collator
-FROM zombie-builder AS test-parachain-undying-collator
-# Polkadot git tag to fetch
-ARG POLKADOT_RELEASE
-RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git
-WORKDIR /home/nonroot/polkadot
-RUN cargo build -p test-parachain-undying-collator
+FROM zombie-builder AS zombie-builder-polkadot-sdk
+ARG POLKADOT_SDK_RELEASE
+WORKDIR /home/nonroot/
+RUN git clone --depth 1 --branch $POLKADOT_SDK_RELEASE https://github.com/paritytech/polkadot-sdk.git
 
-# Image with polkadot-test-malus
-FROM zombie-builder AS polkadot-test-malus
-# Polkadot git tag to fetch
-ARG POLKADOT_RELEASE
-RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git
-WORKDIR /home/nonroot/polkadot
-RUN cargo build -p polkadot-test-malus
+FROM zombie-builder-polkadot-sdk AS zombie-builder-polkadot-sdk-bin
+WORKDIR /home/nonroot/polkadot-sdk/
+RUN cargo update -p test-parachain-adder-collator \
+    -p polkadot-test-malus \
+    -p test-parachain-undying-collator
+RUN cargo build --profile testnet -p test-parachain-adder-collator \
+    -p polkadot-test-malus \
+    -p test-parachain-undying-collator
+RUN find /home/nonroot/polkadot-sdk/target/ -maxdepth 2 -print
 
-# Image with polkadot
 FROM docker.io/parity/polkadot:$POLKADOT_RELEASE_GLOBAL AS polkadot
 
+FROM docker.io/parity/polkadot-parachain:$POLKADOT_RELEASE_GLOBAL_NUMERIC AS polkadot-parachain
+
 FROM zombie-builder AS final
+
 RUN mkdir -p /home/nonroot/bin
-COPY --from=polkadot-parachain /home/nonroot/cumulus/target/release/polkadot-parachain /home/nonroot/bin
-COPY --from=test-parachain /home/nonroot/cumulus/target/release/test-parachain /home/nonroot/bin
-COPY --from=test-parachain /home/nonroot/cumulus/target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /home/nonroot/bin
-COPY --from=test-parachain-adder-collator /home/nonroot/polkadot/target/debug/adder-collator /home/nonroot/bin
-COPY --from=test-parachain-undying-collator /home/nonroot/polkadot/target/debug/undying-collator /home/nonroot/bin
-COPY --from=polkadot-test-malus /home/nonroot/polkadot/target/debug/malus /home/nonroot/bin
+COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/malus /home/nonroot/bin
+COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/adder-collator /home/nonroot/bin
+COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/undying-collator /home/nonroot/bin
 COPY --from=polkadot /usr/bin/polkadot /home/nonroot/bin
+COPY --from=polkadot-parachain /usr/local/bin/polkadot-parachain /home/nonroot/bin
+COPY polkadot-sdk-versions.txt /home/nonroot/polkadot-sdk-versions.txt
+COPY zombienet-versions.txt /home/nonroot/zombienet-versions.txt
 ENV PATH=/home/nonroot/bin:$PATH
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget && \
+    apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget nano && \
     curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
     echo \
       "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \
     wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \  
     add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \
     apt-get update && \
-    apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 nano && \
+    apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://github.com/paritytech/zombienet/releases/download/v1.3.55/zombienet-linux-x64 && \
+ARG ZOMBIENET_RELEASE
+RUN wget https://github.com/paritytech/zombienet/releases/download/$ZOMBIENET_RELEASE/zombienet-linux-x64 && \
     chmod +x zombienet-linux-x64 && \
     cp zombienet-linux-x64 /home/nonroot/bin && \
     rm -rf zombienet-linux-x64

zombienet/docker/tester.Dockerfile

@@ -1,13 +1,10 @@
-#Image with kagome
+ARG POLKADOT_SDK_RELEASE
+
 FROM qdrvm/kagome:latest as kagome
 
-FROM qdrvm/zombie-builder:latest AS tester
+FROM qdrvm/zombie-builder:$POLKADOT_SDK_RELEASE AS tester
 COPY --from=kagome /usr/local/bin/kagome /home/nonroot/bin
 RUN git clone https://github.com/qdrvm/kagome.git
-RUN cp /home/nonroot/bin/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm \
-        /home/nonroot/kagome/zombienet/0004-runtime-upgrade/ && \
-    cp /home/nonroot/bin/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm \
-        /home/nonroot/kagome/zombienet/0004-runtime-upgrade-kagome/
 RUN mkdir /home/nonroot/.local && \
    chown nonroot:nonroot /home/nonroot/.local && \
    chown nonroot:nonroot /tmp

zombienet/docker/version.py

@@ -1,61 +1,90 @@
-import re, sys, git
+import re, sys, subprocess
 
-def write_file(file:str, text:str):
-  filename = file + "-version.txt"
-  f = open(filename, "w")
-  f.write(file.upper() + "_RELEASE=" + text)
-  f.close()
+def list_all_tags_for_remote_git_repo(repo_url):
+  result = subprocess.run([
+    "git", "ls-remote", "--tags", repo_url
+  ], stdout=subprocess.PIPE, text=True)
+  output_lines = result.stdout.splitlines()
+  tags = [
+    line.split("refs/tags/")[-1] for line in output_lines
+    if "refs/tags/" in line and "^{}" not in line
+  ]
+  return tags
+
+def write_file(file: str, text: str):
+  version_info = {
+    "version": text,
+    "short_version": text.replace("polkadot-", ""),
+    "numeric_version": re.sub(r'^(polkadot-)?v', '', text),
+    "polkadot_format_version": text if "polkadot-" in text else None
+  }
+
+  filename = file + "-versions.txt"
+  with open(filename, "w") as f:
+    for key, value in version_info.items():
+      f.write(f"{key}: {value}\n")
 
 def get_last_tag(release_tags):
-  int_tags = []
-  for i in range(len(release_tags)):
-    temp = release_tags[i].split(".")
-    int_tags.append([int(temp[0]),int(temp[1]),int(temp[2])])
-  last_tag_int = sorted(int_tags)[-1]
-  last_tag_str = "v" + str(last_tag_int[0]) + "." + \
-                       str(last_tag_int[1]) + "." + \
-                       str(last_tag_int[2])
-  return last_tag_str
+  def version_key(tag):
+    parts = re.sub(r'^(polkadot-)?v', '', tag).split(".")
+    return [int(part) for part in parts]
+  filtered_tags = [tag for tag in release_tags if re.match(r'^(polkadot-)?v\d+\.\d+\.\d+$', tag)]
+  sorted_tags = sorted(filtered_tags, key=version_key)
+  return sorted_tags[-1] if sorted_tags else None
 
 def select_release_tags(all_tags):
   tags = []
   for tag in all_tags:
-    res = re.search(r'^v([\d]*.[\d]*.[\d]*)$', tag.name)
-    if res != None:
-      tags.append(res.group(1))
+    res = re.search(r'^(polkadot-)?v(\d+\.\d+\.\d+)$', tag)
+    if res is not None:
+      tags.append(res.group(0))
   return tags
 
-def get_version(repo_name):
-  git.Git(".").clone(repo_name)
-  repo_short_name = repo_name.split("/")[-1].split(".")[0]
-  cloned_repo = git.Repo(repo_short_name)
-  all_tags = cloned_repo.tags
+def get_version(repo_url):
+  repo_short_name = repo_url.split("/")[-1].split(".")[0]
+  all_tags = list_all_tags_for_remote_git_repo(repo_url) #cloned_repo.tags
+  print(f"All tags received: {all_tags}")
   release_tags = select_release_tags(all_tags)
+  print((f"Filtered release tags: {release_tags}"))
   last_tag = get_last_tag(release_tags)
+  print (f"Latest release: {last_tag}")
+
   write_file(repo_short_name, last_tag)
 
 def help():
   print("""
     This script: 
-      - takes a repo, 
-      - clones the repo
-      - parses tags and finds the latest release tag with semantic version (v0.0.0)
-      - writes latest tag to file
+      - retrieves tags from a remote Git repository,
+      - filters and sorts these tags to identify the latest release tag based on semantic versioning (e.g., polkadot-v0.0.0),
+      - writes detailed version information to a file.
+
     Usage:
-      python version.py https://github.com/paritytech/polkadot.git
+      python version.py <repository_url>
+      - <repository_url>: URL of the Git repository to process (e.g., https://github.com/paritytech/polkadot.git).
+
     Result:
-      [repo]-version.txt with [REPO]_RELEASE=[version] content
+      The script generates a file named [repo_short_name]-versions.txt containing key-value pairs of version information:
+        - 'version': the latest version tag (e.g., polkadot-v0.1.6),
+        - 'short_version': the short version string (e.g., v0.1.6),
+        - 'numeric_version': the numeric part of the version (e.g., 0.1.6),
+        - 'polkadot_format_version': the complete version tag if it includes 'polkadot-' prefix, otherwise 'None'.
+
     Example:
-      polkadot-version.txt file with POLKADOT_RELEASE=v0.9.43 content for 26.06.2023
+      For a repository URL https://github.com/paritytech/polkadot.git,
+      the script will generate a file named 'polkadot-versions.txt' with content like:
+        version: polkadot-v0.1.6
+        short_version: v0.1.6
+        numeric_version: 0.1.6
+        polkadot_format_version: polkadot-v0.1.6
     """)
 
 def main():
   args = sys.argv[1:]
   if "--help" in args:
-     help()
-     exit()
+    help()
+    exit()
 
   if len(args) == 1:
     get_version(args[0])
-  
+
 main()