diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml index 027a194740..39ee5d8704 100644 --- a/.github/workflows/zombie-builder.yml +++ b/.github/workflows/zombie-builder.yml @@ -6,8 +6,8 @@ # Update zombie builder image each Sunday night on: - schedule: - - cron: '0 2 * * 0' +# schedule: +# - cron: '0 2 * * 0' workflow_dispatch: jobs: diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile index a1524bff2f..6e80c837e6 100644 --- a/zombienet/docker/Makefile +++ b/zombienet/docker/Makefile @@ -1,23 +1,45 @@ builder_last: - python version.py https://github.com/paritytech/polkadot.git && \ - python version.py https://github.com/paritytech/cumulus.git && \ + python version.py https://github.com/paritytech/polkadot-sdk && \ + python version.py https://github.com/paritytech/zombienet && \ make fetch_and_run fetch_and_run: - export $(shell [ ! -n "polkadot-version.txt" ] || cat polkadot-version.txt | grep -v \ - --perl-regexp '^('$$(env | sed 's/=.*//'g | tr '\n' '|')')\=') && \ - export $(shell [ ! -n "cumulus-version.txt" ] || cat cumulus-version.txt | grep -v \ - --perl-regexp '^('$$(env | sed 's/=.*//'g | tr '\n' '|')')\=') && \ - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=$$CUMULUS_RELEASE --build-arg POLKADOT_RELEASE=$$POLKADOT_RELEASE --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE . + if [ -f polkadot-sdk-versions.txt ]; then \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ + DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE -t qdrvm/zombie-builder:latest \ + -f builder.Dockerfile \ + --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ + --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \ + --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \ + --build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \ + else \ + echo "One or more files are missing."; \ + fi builder_push: - docker push qdrvm/zombie-builder:latest + if [ -f polkadot-sdk-versions.txt ]; then \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + docker push qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE ; \ + docker push qdrvm/zombie-builder:latest ; \ + else \ + echo "One or more files are missing."; \ + fi builder: DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=v0.9.420 --build-arg POLKADOT_RELEASE=v0.9.42 --build-arg POLKADOT_RELEASE_GLOBAL=v0.9.42 . tester: - docker build --no-cache -t qdrvm/zombie-tester:latest -f tester.Dockerfile . + python version.py https://github.com/paritytech/polkadot-sdk && \ + if [ -f polkadot-sdk-versions.txt ]; then \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + DOCKER_BUILDKIT=0 docker build --no-cache -t qdrvm/zombie-tester:latest -f tester.Dockerfile \ + --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ + else \ + echo "One or more files are missing."; \ + fi tests: docker run -it qdrvm/zombie-tester:latest diff --git a/zombienet/docker/builder.Dockerfile b/zombienet/docker/builder.Dockerfile index 255e125691..f64f50ad17 100644 --- a/zombienet/docker/builder.Dockerfile +++ b/zombienet/docker/builder.Dockerfile @@ -1,17 +1,17 @@ -# Polkadot docker image to fetch ARG POLKADOT_RELEASE_GLOBAL +ARG POLKADOT_RELEASE_GLOBAL_NUMERIC +ARG ZOMBIENET_RELEASE -# Base image with all dependencies FROM node:18-bullseye-slim as zombie-builder USER root RUN apt-get update && apt update && \ - apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev && \ - apt install -y --no-install-recommends clang ca-certificates openssl && \ + apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev time && \ + apt install -y --no-install-recommends clang ca-certificates openssl && \ rm -rf /var/lib/apt/lists/* && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/root/.cargo/bin:$PATH -RUN rustup default stable && rustup target add wasm32-unknown-unknown +RUN rustup default stable && rustup target add wasm32-unknown-unknown && rustup component add rust-src RUN groupadd --gid 10001 nonroot && \ useradd --home-dir /home/nonroot \ --create-home \ @@ -21,72 +21,50 @@ RUN groupadd --gid 10001 nonroot && \ --uid 10000 nonroot WORKDIR /home/nonroot/ -# Image with polkadot-parachain binary -FROM zombie-builder AS polkadot-parachain -# Cumulus git tag to fetch -ARG CUMULUS_RELEASE -RUN git clone --depth 1 --branch $CUMULUS_RELEASE https://github.com/paritytech/cumulus.git -WORKDIR /home/nonroot/cumulus -RUN cargo build --release --bin polkadot-parachain - -# Image with polkadot-parachain binary -FROM zombie-builder AS test-parachain -# Cumulus git tag to fetch -ARG CUMULUS_RELEASE -RUN git clone --depth 1 --branch $CUMULUS_RELEASE https://github.com/paritytech/cumulus.git -WORKDIR /home/nonroot/cumulus -RUN cargo build --release --locked --bin test-parachain - -# Image with test-parachain-adder-collator binary -FROM zombie-builder AS test-parachain-adder-collator -# Polkadot git tag to fetch -ARG POLKADOT_RELEASE -RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git -WORKDIR /home/nonroot/polkadot -RUN cargo build -p test-parachain-adder-collator - -# Image with test-parachain-undying-collator -FROM zombie-builder AS test-parachain-undying-collator -# Polkadot git tag to fetch -ARG POLKADOT_RELEASE -RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git -WORKDIR /home/nonroot/polkadot -RUN cargo build -p test-parachain-undying-collator +FROM zombie-builder AS zombie-builder-polkadot-sdk +ARG POLKADOT_SDK_RELEASE +WORKDIR /home/nonroot/ +RUN git clone --depth 1 --branch $POLKADOT_SDK_RELEASE https://github.com/paritytech/polkadot-sdk.git -# Image with polkadot-test-malus -FROM zombie-builder AS polkadot-test-malus -# Polkadot git tag to fetch -ARG POLKADOT_RELEASE -RUN git clone --depth 1 --branch $POLKADOT_RELEASE https://github.com/paritytech/polkadot.git -WORKDIR /home/nonroot/polkadot -RUN cargo build -p polkadot-test-malus +FROM zombie-builder-polkadot-sdk AS zombie-builder-polkadot-sdk-bin +WORKDIR /home/nonroot/polkadot-sdk/ +RUN cargo update -p test-parachain-adder-collator \ + -p polkadot-test-malus \ + -p test-parachain-undying-collator +RUN cargo build --profile testnet -p test-parachain-adder-collator \ + -p polkadot-test-malus \ + -p test-parachain-undying-collator +RUN find /home/nonroot/polkadot-sdk/target/ -maxdepth 2 -print -# Image with polkadot FROM docker.io/parity/polkadot:$POLKADOT_RELEASE_GLOBAL AS polkadot +FROM docker.io/parity/polkadot-parachain:$POLKADOT_RELEASE_GLOBAL_NUMERIC AS polkadot-parachain + FROM zombie-builder AS final + RUN mkdir -p /home/nonroot/bin -COPY --from=polkadot-parachain /home/nonroot/cumulus/target/release/polkadot-parachain /home/nonroot/bin -COPY --from=test-parachain /home/nonroot/cumulus/target/release/test-parachain /home/nonroot/bin -COPY --from=test-parachain /home/nonroot/cumulus/target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /home/nonroot/bin -COPY --from=test-parachain-adder-collator /home/nonroot/polkadot/target/debug/adder-collator /home/nonroot/bin -COPY --from=test-parachain-undying-collator /home/nonroot/polkadot/target/debug/undying-collator /home/nonroot/bin -COPY --from=polkadot-test-malus /home/nonroot/polkadot/target/debug/malus /home/nonroot/bin +COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/malus /home/nonroot/bin +COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/adder-collator /home/nonroot/bin +COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/undying-collator /home/nonroot/bin COPY --from=polkadot /usr/bin/polkadot /home/nonroot/bin +COPY --from=polkadot-parachain /usr/local/bin/polkadot-parachain /home/nonroot/bin +COPY polkadot-sdk-versions.txt /home/nonroot/polkadot-sdk-versions.txt +COPY zombienet-versions.txt /home/nonroot/zombienet-versions.txt ENV PATH=/home/nonroot/bin:$PATH RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget && \ + apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget nano && \ curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ echo \ "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \ apt-get update && \ - apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 nano && \ + apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 && \ rm -rf /var/lib/apt/lists/* -RUN wget https://github.com/paritytech/zombienet/releases/download/v1.3.55/zombienet-linux-x64 && \ +ARG ZOMBIENET_RELEASE +RUN wget https://github.com/paritytech/zombienet/releases/download/$ZOMBIENET_RELEASE/zombienet-linux-x64 && \ chmod +x zombienet-linux-x64 && \ cp zombienet-linux-x64 /home/nonroot/bin && \ rm -rf zombienet-linux-x64 diff --git a/zombienet/docker/requirements.txt b/zombienet/docker/requirements.txt deleted file mode 100644 index 814b95c3d7..0000000000 --- a/zombienet/docker/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -GitPython==3.1.41 \ No newline at end of file diff --git a/zombienet/docker/tester.Dockerfile b/zombienet/docker/tester.Dockerfile index 21a3cf36c2..a812af3b89 100644 --- a/zombienet/docker/tester.Dockerfile +++ b/zombienet/docker/tester.Dockerfile @@ -1,13 +1,10 @@ -#Image with kagome +ARG POLKADOT_SDK_RELEASE + FROM qdrvm/kagome:latest as kagome -FROM qdrvm/zombie-builder:latest AS tester +FROM qdrvm/zombie-builder:$POLKADOT_SDK_RELEASE AS tester COPY --from=kagome /usr/local/bin/kagome /home/nonroot/bin RUN git clone https://github.com/qdrvm/kagome.git -RUN cp /home/nonroot/bin/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm \ - /home/nonroot/kagome/zombienet/0004-runtime-upgrade/ && \ - cp /home/nonroot/bin/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm \ - /home/nonroot/kagome/zombienet/0004-runtime-upgrade-kagome/ RUN mkdir /home/nonroot/.local && \ chown nonroot:nonroot /home/nonroot/.local && \ chown nonroot:nonroot /tmp diff --git a/zombienet/docker/version.py b/zombienet/docker/version.py index 3d687f5d8e..d6e8e74f45 100644 --- a/zombienet/docker/version.py +++ b/zombienet/docker/version.py @@ -1,61 +1,90 @@ -import re, sys, git +import re, sys, subprocess -def write_file(file:str, text:str): - filename = file + "-version.txt" - f = open(filename, "w") - f.write(file.upper() + "_RELEASE=" + text) - f.close() +def list_all_tags_for_remote_git_repo(repo_url): + result = subprocess.run([ + "git", "ls-remote", "--tags", repo_url + ], stdout=subprocess.PIPE, text=True) + output_lines = result.stdout.splitlines() + tags = [ + line.split("refs/tags/")[-1] for line in output_lines + if "refs/tags/" in line and "^{}" not in line + ] + return tags + +def write_file(file: str, text: str): + version_info = { + "version": text, + "short_version": text.replace("polkadot-", ""), + "numeric_version": re.sub(r'^(polkadot-)?v', '', text), + "polkadot_format_version": text if "polkadot-" in text else None + } + + filename = file + "-versions.txt" + with open(filename, "w") as f: + for key, value in version_info.items(): + f.write(f"{key}: {value}\n") def get_last_tag(release_tags): - int_tags = [] - for i in range(len(release_tags)): - temp = release_tags[i].split(".") - int_tags.append([int(temp[0]),int(temp[1]),int(temp[2])]) - last_tag_int = sorted(int_tags)[-1] - last_tag_str = "v" + str(last_tag_int[0]) + "." + \ - str(last_tag_int[1]) + "." + \ - str(last_tag_int[2]) - return last_tag_str + def version_key(tag): + parts = re.sub(r'^(polkadot-)?v', '', tag).split(".") + return [int(part) for part in parts] + filtered_tags = [tag for tag in release_tags if re.match(r'^(polkadot-)?v\d+\.\d+\.\d+$', tag)] + sorted_tags = sorted(filtered_tags, key=version_key) + return sorted_tags[-1] if sorted_tags else None def select_release_tags(all_tags): tags = [] for tag in all_tags: - res = re.search(r'^v([\d]*.[\d]*.[\d]*)$', tag.name) - if res != None: - tags.append(res.group(1)) + res = re.search(r'^(polkadot-)?v(\d+\.\d+\.\d+)$', tag) + if res is not None: + tags.append(res.group(0)) return tags -def get_version(repo_name): - git.Git(".").clone(repo_name) - repo_short_name = repo_name.split("/")[-1].split(".")[0] - cloned_repo = git.Repo(repo_short_name) - all_tags = cloned_repo.tags +def get_version(repo_url): + repo_short_name = repo_url.split("/")[-1].split(".")[0] + all_tags = list_all_tags_for_remote_git_repo(repo_url) #cloned_repo.tags + print(f"All tags received: {all_tags}") release_tags = select_release_tags(all_tags) + print((f"Filtered release tags: {release_tags}")) last_tag = get_last_tag(release_tags) + print (f"Latest release: {last_tag}") + write_file(repo_short_name, last_tag) def help(): print(""" This script: - - takes a repo, - - clones the repo - - parses tags and finds the latest release tag with semantic version (v0.0.0) - - writes latest tag to file + - retrieves tags from a remote Git repository, + - filters and sorts these tags to identify the latest release tag based on semantic versioning (e.g., polkadot-v0.0.0), + - writes detailed version information to a file. + Usage: - python version.py https://github.com/paritytech/polkadot.git + python version.py + - : URL of the Git repository to process (e.g., https://github.com/paritytech/polkadot.git). + Result: - [repo]-version.txt with [REPO]_RELEASE=[version] content + The script generates a file named [repo_short_name]-versions.txt containing key-value pairs of version information: + - 'version': the latest version tag (e.g., polkadot-v0.1.6), + - 'short_version': the short version string (e.g., v0.1.6), + - 'numeric_version': the numeric part of the version (e.g., 0.1.6), + - 'polkadot_format_version': the complete version tag if it includes 'polkadot-' prefix, otherwise 'None'. + Example: - polkadot-version.txt file with POLKADOT_RELEASE=v0.9.43 content for 26.06.2023 + For a repository URL https://github.com/paritytech/polkadot.git, + the script will generate a file named 'polkadot-versions.txt' with content like: + version: polkadot-v0.1.6 + short_version: v0.1.6 + numeric_version: 0.1.6 + polkadot_format_version: polkadot-v0.1.6 """) def main(): args = sys.argv[1:] if "--help" in args: - help() - exit() + help() + exit() if len(args) == 1: get_version(args[0]) - + main() \ No newline at end of file