From 7b4f60817f0674963949384b6ad5be25c2d122a5 Mon Sep 17 00:00:00 2001 From: Niccolo Raspa <6024049+niccoloraspa@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:06:27 +0200 Subject: [PATCH] Add script to generate cosmovisor binaries json and list osmosis upgrades (#5907) * Add script to generate cosmovisor binaries json and improve upgrade docs * Linting --- networks/osmosis-1/README.md | 256 ++++++++++++++++++++ scripts/release/create_all_binaries_json.sh | 26 ++ scripts/release/create_binaries_json.py | 118 +++++++++ scripts/release/requirements.txt | 5 + 4 files changed, 405 insertions(+) create mode 100644 networks/osmosis-1/README.md create mode 100755 scripts/release/create_all_binaries_json.sh create mode 100644 scripts/release/create_binaries_json.py create mode 100644 scripts/release/requirements.txt diff --git a/networks/osmosis-1/README.md b/networks/osmosis-1/README.md new file mode 100644 index 00000000000..f10caf577e8 --- /dev/null +++ b/networks/osmosis-1/README.md @@ -0,0 +1,256 @@ +# Upgrade History + +The following table presents a list of the osmosis versions. + +Each version is identified by a specific id, name, tag, block height and software upgrade proposal. + +| ID | Name | Tag | Starting Block | Release | Proposal | +|-------|-----------|-----------|----------------|--------------------------------------------------------------------------|------------------------------------------------------| +| `v3` | Lithium | `v3.1.0` | 0 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v3.1.0/) | N.A. (Genesis) | +| `v4` | Berylium | `v4.2.0` | 1314500 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v4.2.0/) | [38](https://www.mintscan.io/osmosis/proposals/38) | +| `v5` | Boron | `v6.4.0` | 2383300 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v6.4.0) | [95](https://www.mintscan.io/osmosis/proposals/95) | +| `v7` | Carbon | `v8.0.0` | 3401000 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v8.0.0/) | [157](https://www.mintscan.io/osmosis/proposals/157) | +| `v9` | Nitrogen | `v10.1.1` | 4707300 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v10.1.1/) | [252](https://www.mintscan.io/osmosis/proposals/252) | +| `v11` | | `v11.0.1` | 5432450 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v11.0.1/) | [296](https://www.mintscan.io/osmosis/proposals/296) | +| `v12` | Oxygen | `v12.3.0` | 6246000 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v12.3.0/) | [335](https://www.mintscan.io/osmosis/proposals/335) | +| `v13` | Fluorine | `v13.1.2` | 7241500 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v13.1.2/) | [370](https://www.mintscan.io/osmosis/proposals/370) | +| `v14` | Neon | `v14.0.1` | 7937500 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v14.0.1/) | [401](https://www.mintscan.io/osmosis/proposals/401) | +| `v15` | Sodium | `v15.2.0` | 8732500 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v15.2.0/) | [458](https://www.mintscan.io/osmosis/proposals/458) | +| `v16` | Magnesium | `v16.1.1` | 10517000 | [Release](https://github.com/osmosis-labs/osmosis/releases/tag/v16.1.1/) | [556](https://www.mintscan.io/osmosis/proposals/556) | + +## Upgrade Binaries + +### v3.1.0 + +```json +{ + "binaries": { + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v3.1.0/osmosisd-3.1.0-darwin-amd64?checksum=sha256:a532f25ae754d2573f6a3c91ba59496ddb9f6766ccf6f69f408f6e1597144a74", + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v3.1.0/osmosisd-3.1.0-linux-amd64?checksum=sha256:6a73d75e9c75ea402c13edc8c5c4ed08e26c5d8e517d540a9ca8b7e7afa67f79", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v3.1.0/osmosisd-3.1.0-linux-arm64?checksum=sha256:893f8a9786ae76d4217260201cd94ab67010f68d98b9676a9b31c0a5e68d1eae" + } +} +``` + +### v4.2.0 + +```json +{ + "binaries": { + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v4.2.0/osmosisd-4.2.0-darwin-amd64?checksum=sha256:eee08350b223dd06a2aa16aab44aa51eb116f6267924ee1e788ca28fb54fe02d", + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v4.2.0/osmosisd-4.2.0-linux-amd64?checksum=sha256:a11c61a737983d176f23ce83fa5ff985000ce8d5107d738ee6fa7d59b8dd3053", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v4.2.0/osmosisd-4.2.0-linux-arm64?checksum=sha256:41260be15e874fbc6cc49757d9fe3d4e459634729e2b745923e508e9cb26f837" + } +} +``` + +### v6.4.0 + +```json +{ + "binaries": { + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v6.4.0/osmosisd-6.4.0-darwin-amd64?checksum=sha256:735c7828b0bc311381f4c18081fa648f849df03aeccf173425cc52a634e3c7d8", + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v6.4.0/osmosisd-6.4.0-linux-amd64?checksum=sha256:e4017da5d1a0a3b37b4f6936ba7ef16f39972ae25f95feae43e506f14933cf94", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v6.4.0/osmosisd-6.4.0-linux-arm64?checksum=sha256:a101bb3feb0419293a3ecee17d732a312bf9e864a829905ed509c65b5944040b" + } +} +``` + +### v8.0.0 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v8.0.0/osmosisd-8.0.0-linux-amd64?checksum=sha256:4559ffe7d1e83b1519c2d45a709d35a89b51f8b35f8bba3b58aef92e667e254c" + } +} +``` + +### v10.1.1 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v10.1.1/osmosisd-10.1.1-linux-amd64?checksum=sha256:aeae58f8b0be86d5e6e3aec1a8774eab4947207c88c7d4f309c46da98f6694e8", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v10.1.1/osmosisd-10.1.1-linux-arm64?checksum=sha256:d2c672ffa9782687f91d8d03bd23fdf8bd2fbe8b79c9cfcf8e9d302a1238a12c" + } +} +``` + +### v11.0.1 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v11.0.1/osmosisd-11.0.1-linux-amd64?checksum=sha256:41b8fd2345a5e5b77ee5ed9b9ec5370d94bd1b1aa0d4ac2ac0ab02ee98ddd0d8", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v11.0.1/osmosisd-11.0.1-linux-arm64?checksum=sha256:267776170495ecaa831238ea8994f8790a379663c9ae47a2e93e5beceafd8e1d" + } +} +``` + +### v12.3.0 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v12.3.0/osmosisd-12.3.0-linux-amd64?checksum=sha256:958210c919d13c281896fa9773c323c5534f0fa46d74807154f737609a00db70", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v12.3.0/osmosisd-12.3.0-linux-arm64?checksum=sha256:a931618c8a839c30e5cecfd2a88055cda1d68cc68557fe3303fe14e2de3bef8f" + } +} +``` + +### v13.1.2 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v13.1.2/osmosisd-13.1.2-linux-amd64?checksum=sha256:67ed53046667c72ec6bfe962bcb4d6b122610876b3adf75fb7820ce52c34872d", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v13.1.2/osmosisd-13.1.2-linux-arm64?checksum=sha256:ad35c2a8d55852fa28187a55bdeb983494c07923f2a8a9f4479fb044d8d62bd9" + } +} +``` + +### v14.0.1 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v14.0.1/osmosisd-14.0.1-linux-amd64?checksum=sha256:2cc4172bcf000f0f06b30b16864d875a8de2ee12df994a593dfd52a506851bce", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v14.0.1/osmosisd-14.0.1-linux-arm64?checksum=sha256:9a44c17d239c8d9afd19d0ff0bd14ca883fb9e9fbf69aff18c2607ffa6bff378" + } +} +``` + +### v15.2.0 + +```json +{ + "binaries": { + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v15.2.0/osmosisd-15.2.0-linux-amd64?checksum=sha256:3aab2f2668cb5a713d5770e46a777ef01c433753378702d9ae941aa2d1ee5618", + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v15.2.0/osmosisd-15.2.0-linux-arm64?checksum=sha256:e158d30707a0ea51482237f99676223e81ce5a353966a5c83791d2662a930f35" + } +} +``` + +### v16.1.1 + +```json +{ + "binaries": { + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/osmosisd-16.1.1-linux-arm64?checksum=sha256:b96ff1f4c9b4abecb1b38998b1a1f891cfed2cc8078ab64914b151183c0c199b", + "darwin/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/osmosisd-16.1.1-darwin-arm64?checksum=sha256:c743da4d3632a2bc3ea0ce784bbd13383492a4a34d53295eb2c96987bacf8e8c", + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/osmosisd-16.1.1-darwin-amd64?checksum=sha256:d856ebda9c31f052d10a78443967a93374f2033292f0afdb6434b82b4ed79790", + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/osmosisd-16.1.1-linux-amd64?checksum=sha256:f838618633c1d42f593dc33d26b25842f5900961e987fc08570bb81a062e311d" + } +} +``` + +## Replay from Genesis using Cosmovisor + +Assuming that your osmosis home it's already initialized with the desired genesis and configuration, +to replay the chain from genesis using Cosmovisor: + +1. Install version `v1.2.0` from the official [repository](https://github.com/cosmos/cosmos-sdk/tree/main/tools/cosmovisor). + +Alternatively, you can download the appropriate binary for your platform from our mirrors: + +| Platform | Architecture | Cosmovisor Binary URL | +|----------|--------------|------------------------------------------------------------------------------------------------------------| +| darwin | amd64 | [Download](https://osmosis.fra1.digitaloceanspaces.com/binaries/cosmovisor/cosmovisor-v1.2.0-darwin-amd64) | +| darwin | arm64 | [Download](https://osmosis.fra1.digitaloceanspaces.com/binaries/cosmovisor/cosmovisor-v1.2.0-darwin-arm64) | +| linux | amd64 | [Download](https://osmosis.fra1.digitaloceanspaces.com/binaries/cosmovisor/cosmovisor-v1.2.0-linux-amd64) | +| linux | arm64 | [Download](https://osmosis.fra1.digitaloceanspaces.com/binaries/cosmovisor/cosmovisor-v1.2.0-linux-arm64) | + +1. Initialize the Cosmovisor directory following the specific structure outlined below: + +```bash + + ├── genesis + │ └── bin + │ └── osmosisd + └── upgrades + ├── v11 + │ └── bin + │ └── osmosisd + ├── v12 + │ └── bin + │ └── osmosisd + ├── v13 + │ └── bin + │ └── osmosisd + ├── v14 + │ └── bin + │ └── osmosisd + ├── v15 + │ └── bin + │ └── osmosisd + ├── v16 + │ └── bin + │ └── osmosisd + ├── v4 + │ └── bin + │ └── osmosisd + ├── v5 + │ └── bin + │ └── osmosisd + ├── v7 + │ └── bin + │ └── osmosisd + └── v9 + └── bin + └── osmosisd +``` + +You can utilize the provided script to download the required binaries and initialize the Cosmovisor directory: + +```bash +# Define osmosis home +osmosis_home="$HOME/.osmosisd" + +# List of versions and their URLs +versions_info=( + "v3:https://github.com/osmosis-labs/osmosis/releases/download/v3.1.0/osmosisd-3.1.0-linux-amd64?checksum=sha256:6a73d75e9c75ea402c13edc8c5c4ed08e26c5d8e517d540a9ca8b7e7afa67f79" + "v4:https://github.com/osmosis-labs/osmosis/releases/download/v4.2.0/osmosisd-4.2.0-linux-amd64?checksum=sha256:a11c61a737983d176f23ce83fa5ff985000ce8d5107d738ee6fa7d59b8dd3053" + "v5:https://github.com/osmosis-labs/osmosis/releases/download/v6.4.0/osmosisd-6.4.0-linux-amd64?checksum=sha256:e4017da5d1a0a3b37b4f6936ba7ef16f39972ae25f95feae43e506f14933cf94" + "v7:https://github.com/osmosis-labs/osmosis/releases/download/v8.0.0/osmosisd-8.0.0-linux-amd64?checksum=sha256:4559ffe7d1e83b1519c2d45a709d35a89b51f8b35f8bba3b58aef92e667e254c" + "v9:https://github.com/osmosis-labs/osmosis/releases/download/v10.1.1/osmosisd-10.1.1-linux-amd64?checksum=sha256:aeae58f8b0be86d5e6e3aec1a8774eab4947207c88c7d4f309c46da98f6694e8" + "v11:https://github.com/osmosis-labs/osmosis/releases/download/v11.0.1/osmosisd-11.0.1-linux-amd64?checksum=sha256:41b8fd2345a5e5b77ee5ed9b9ec5370d94bd1b1aa0d4ac2ac0ab02ee98ddd0d8" + "v12:https://github.com/osmosis-labs/osmosis/releases/download/v12.3.0/osmosisd-12.3.0-linux-amd64?checksum=sha256:958210c919d13c281896fa9773c323c5534f0fa46d74807154f737609a00db70" + "v13:https://github.com/osmosis-labs/osmosis/releases/download/v13.1.2/osmosisd-13.1.2-linux-amd64?checksum=sha256:67ed53046667c72ec6bfe962bcb4d6b122610876b3adf75fb7820ce52c34872d" + "v14:https://github.com/osmosis-labs/osmosis/releases/download/v14.0.1/osmosisd-14.0.1-linux-amd64?checksum=sha256:2cc4172bcf000f0f06b30b16864d875a8de2ee12df994a593dfd52a506851bce" + "v15:https://github.com/osmosis-labs/osmosis/releases/download/v15.2.0/osmosisd-15.2.0-linux-amd64?checksum=sha256:3aab2f2668cb5a713d5770e46a777ef01c433753378702d9ae941aa2d1ee5618" + "v16:https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/osmosisd-16.1.1-linux-amd64?checksum=sha256:f838618633c1d42f593dc33d26b25842f5900961e987fc08570bb81a062e311d" +) + +# Create the cosmovisor directory +echo "📁 Creating the cosmovisor directory: ${osmosis_home}/cosmovisor" +mkdir -p "${osmosis_home}/cosmovisor" + +# Create the genesis directory and download v3 binary to /cosmovisor/genesis/bin +echo "📁 Creating the genesis directory: ${osmosis_home}/cosmovisor/genesis/bin" +mkdir -p "${osmosis_home}/cosmovisor/genesis/bin" + +echo "⬇️ Downloading v3 binary to: ${osmosis_home}/cosmovisor/genesis/bin/osmosisd" +wget -q -O "${osmosis_home}/cosmovisor/genesis/bin/osmosisd" "$(echo ${versions_info[0]} | cut -d: -f2)" + +# Create the upgrades directories for each version and download the binaries +for version_info in "${versions_info[@]:1}"; do + version="${version_info%%:*}" + binary_url="${version_info#*:}" + echo + echo "📁 Creating ${version} directory: ${osmosis_home}/cosmovisor/upgrades/${version}/bin" + mkdir -p "${osmosis_home}/cosmovisor/upgrades/${version}/bin" + echo "⬇️ Downloading ${version} binary to: ${osmosis_home}/cosmovisor/upgrades/${version}/bin/osmosisd" + wget -q -O "${osmosis_home}/cosmovisor/upgrades/${version}/bin/osmosisd" "$binary_url" +done +``` + +3. Replaying the chain from historical data requires the presence of at least one archive nodes in the persistent peers. +Ensure that you include the following configuration in your `config.toml` file: + +```toml +[p2p] +persistent_peers = "37c195e518c001099f956202d34af029b04f2c97@65.109.20.216:26656" +``` + +4. Run cosmovisor with `DAEMON_ALLOW_DOWNLOAD_BINARIES=false` diff --git a/scripts/release/create_all_binaries_json.sh b/scripts/release/create_all_binaries_json.sh new file mode 100755 index 00000000000..84cadc40bb0 --- /dev/null +++ b/scripts/release/create_all_binaries_json.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +tags=( + "v3.1.0" + "v4.2.0" + "v6.4.0" + "v8.0.0" + "v10.1.1" + "v11.0.1" + "v12.3.0" + "v13.1.2" + "v14.0.1" + "v15.2.0" + "v16.1.1" +) + +echo "# Cosmovisor binaries" + +for tag in ${tags[@]}; do + echo + echo "## ${tag}" + echo + echo '```json' + python create_binaries_json.py --tag $tag + echo '```' +done diff --git a/scripts/release/create_binaries_json.py b/scripts/release/create_binaries_json.py new file mode 100644 index 00000000000..4f7d4476e8b --- /dev/null +++ b/scripts/release/create_binaries_json.py @@ -0,0 +1,118 @@ +""" +Usage: +This script generates a JSON object containing binary download URLs and their corresponding checksums +for a given release tag of osmosis-labs/osmosis or from a provided checksum URL. +The binary JSON is compatible with cosmovisor and with the chain registry. + +You can run this script with the following commands: + +❯ python create_binaries_json.py --checksums_url https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/sha256sum.txt + +Output: +{ + "binaries": { + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-linux-arm64?checksum=", + "darwin/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-darwin-arm64?checksum=", + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-darwin-amd64?checksum=, + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-linux-amd64?checksum=>" + } +} + +Expects a checksum in the form: + + osmosisd---[.tar.gz] + osmosisd---[.tar.gz] +... + +Example: + +f838618633c1d42f593dc33d26b25842f5900961e987fc08570bb81a062e311d osmosisd-16.1.1-linux-amd64 +fa6699a763487fe6699c8720a2a9be4e26a4f45aafaec87aa0c3aced4cbdd155 osmosisd-16.1.1-linux-amd64.tar.gz + +(From: https://github.com/osmosis-labs/osmosis/releases/download/v16.1.1/sha256sum.txt) + +❯ python create_binaries_json.py --tag v16.1.1 + +Output: +{ + "binaries": { + "linux/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-linux-arm64?checksum=", + "darwin/arm64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-darwin-arm64?checksum=", + "darwin/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-darwin-amd64?checksum=", + "linux/amd64": "https://github.com/osmosis-labs/osmosis/releases/download/16.1.1/osmosisd-16.1.1-linux-amd64?checksum=>" + } +} + +Expect a checksum to be present at: +https://github.com/osmosis-labs/osmosis/releases/download//sha256sum.txt +""" + +import requests +import json +import argparse +import re +import sys + +def validate_tag(tag): + pattern = '^v[0-9]+.[0-9]+.[0-9]+$' + return bool(re.match(pattern, tag)) + +def download_checksums(checksums_url): + + response = requests.get(checksums_url) + if response.status_code != 200: + raise ValueError(f"Failed to fetch sha256sum.txt. Status code: {response.status_code}") + return response.text + +def checksums_to_binaries_json(checksums): + + binaries = {} + + # Parse the content and create the binaries dictionary + for line in checksums.splitlines(): + checksum, filename = line.split(' ') + + # exclude tar.gz files + if not filename.endswith('.tar.gz') and filename.startswith('osmosisd'): + try: + _, tag, platform, arch = filename.split('-') + except ValueError: + print(f"Error: Expected binary name in the form: osmosisd-X.Y.Z-platform-architecture, but got {filename}") + sys.exit(1) + _, tag, platform, arch, = filename.split('-') + # exclude universal binaries and windows binaries + if arch == 'all' or platform == 'windows': + continue + binaries[f"{platform}/{arch}"] = f"https://github.com/osmosis-labs/osmosis/releases/download/v{tag}/{filename}?checksum=sha256:{checksum}" + + binaries_json = { + "binaries": binaries + } + + return json.dumps(binaries_json, indent=2) + +def main(): + + parser = argparse.ArgumentParser(description="Create binaries json") + parser.add_argument('--tag', metavar='tag', type=str, help='the tag to use (e.g v16.1.1)') + parser.add_argument('--checksums_url', metavar='checksums_url', type=str, help='URL to the checksum') + + args = parser.parse_args() + + # Validate the tag format + if args.tag and not validate_tag(args.tag): + print("Error: The provided tag does not follow the 'vX.Y.Z' format.") + sys.exit(1) + + # Ensure that only one of --tag or --checksums_url is specified + if not bool(args.tag) ^ bool(args.checksums_url): + parser.error("Only one of tag or --checksums_url must be specified") + sys.exit(1) + + checksums_url = args.checksums_url if args.checksums_url else f"https://github.com/osmosis-labs/osmosis/releases/download/{args.tag}/sha256sum.txt" + checksums = download_checksums(checksums_url) + binaries_json = checksums_to_binaries_json(checksums) + print(binaries_json) + +if __name__ == "__main__": + main() diff --git a/scripts/release/requirements.txt b/scripts/release/requirements.txt new file mode 100644 index 00000000000..8ef629c15f9 --- /dev/null +++ b/scripts/release/requirements.txt @@ -0,0 +1,5 @@ +certifi==2023.7.22 +charset-normalizer==3.2.0 +idna==3.4 +requests==2.31.0 +urllib3==2.0.4