From 7fa6f6ef732e2ba4c533fa6a86ac6ec4af961d7c Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Tue, 20 Feb 2024 20:08:22 +0100 Subject: [PATCH 01/11] Bump RSTUF services - RSTUF API v0.12.0b1 - RSTUF Worker v0.14.0b1 Signed-off-by: Kairo Araujo --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c525be9c8855..77538326b9eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,7 +160,7 @@ services: SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}" rstuf-api: - image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.9.0b1 + image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.12.0b1 ports: - 8001:80 environment: @@ -170,7 +170,7 @@ services: - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 rstuf-worker: - image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.11.0b1 + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.14.0b1 volumes: - rstuf-metadata:/var/opt/repository-service-tuf/storage environment: From 8e050317ced03bc747d069e800416559881e8d3f Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Tue, 20 Feb 2024 20:09:20 +0100 Subject: [PATCH 02/11] Add RSTUF keys for development enviroment Include RSTUF keys to be used on development environment. Include two root keys: root1: ecdsa format root2: ed25519 format Include online key 2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241 RSA Signed-off-by: Kairo Araujo Signed-off-by: Lukas Puehringer --- ...3ab086b3def14c89d24eef18f49c32508c2f60e241 | 28 +++++++++++++++++++ dev/rstuf/keys/root/root1 | 5 ++++ dev/rstuf/keys/root/root2 | 3 ++ 3 files changed, 36 insertions(+) create mode 100644 dev/rstuf/keys/online/2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241 create mode 100644 dev/rstuf/keys/root/root1 create mode 100644 dev/rstuf/keys/root/root2 diff --git a/dev/rstuf/keys/online/2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241 b/dev/rstuf/keys/online/2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241 new file mode 100644 index 000000000000..82406424dc8a --- /dev/null +++ b/dev/rstuf/keys/online/2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241 @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQDCFfquKiIv9xfl +izfYicVTnYfy0vXhB5fS1pxl1v4DBwfqciH3uHdFIe8YPtfLq5oxGYaV0CdJVjAk +eqgnmPbKdXNT9EWmv7qfw+zRAWDkaaUSplugWMttLRuxFBxdeeok2xUsQ9PdQu+7 +WsVFtbMAwWM2002Tlb+7QYTNefcPTeMh2EPwgoB2tNF/RUwP1NEJct0yYRTa+R/U +1VSc5Nv91FflADxO0HKVWei5mSdtq9DQn+k+Uct/FSCwrOh3AOUmZs8b0FbBepAK +Y/QOA4Y0hnVfbzUw4FoZuQ1US6e02MJJpDFCN8AtVVTk0B9qXZzAwg22pqhztZr5 +Na+kUQhbAgMBAAECggEAFZPH+NDqWBbKa1Sc8s/uRit/T7mwaEIl2OTPImtSdhe0 +A5aIvDef2um44SMrbpM3YzoJQmKP25FfbM7OHwjcdwmztqOzkqRCJTzs+ReEJCCy +n24rRZpZk1uudnNb6/B/3XUV14P66+BjMpsWz3cx3WWimBfJyhyd4j2YfBeRJfw7 +GLCJ0Jeplj0hKEC4Yo6Dvppnl0DJn8NnsnXLRTwepjwB/EpSxnrpzwBBwzsMTcx/ +2zKC9sZhTE1RDsgbw2IIUiBk1enAhZtmiS/BFT9Y4jWaeXTkkVSnFXPZLYPkdB9W +sHgKGiWOSX/1j90IHaKsSKRFUdn3FHtDTde7o4kGQQKBgQDtdS+WBHfVvBH9iQgw +GWc3KKJPcKHC1m4+GOHhIElb0f5l/y6OTZkvK/bPtKJ8bpufsr9jBVQYubIVfJ2j +ZmO0ukclkzIjzwvY9sSHbnWzFfKbjqNG1zGaZYNe0WM/Lx51pG69hxlVzivLAObf +fqYR0+dt5imD/46FcfHFkTQdCwKBgQDRPchsq4zxxvqMYGxzMfyp7l8y1lLcORHS +j2qkOB0n973DggW2sLIEl3uqf/schpbYO8zFs/1YKrJ5LNnYF14GduugmS5znpnb +YvMJyTXFAqmcbl48ahVUvyOrgxTAOJOfFLRXwZiIVzaAaOop+Ph6A4hEYvXWJ8FW +j6lVr7mz8QKBgFYabArly95At/VLPyDR1U92+IP9v2o6/vadZyqO3org9nJdua/4 +C1fDhVeDlHeyU9PwqN1rDTd5/k00RqT9d6IM+cdyPHgnl5AwysqhDyTFDJfDfQku +9tmZfa1gF7DNkSnvWgh3eIRYoiCWTyEzd1x3ji+Xie5HOJLC4nxVTqRJAoGAZQb6 +rZWLAPX85ShtVJVvFDFW37nh2hjoBQ1gBRhe43xXsH0n+xSHb3YgrKsMeLJ3RMJi +1ZZZHWfIMn+4UwC9Uku66xjq98I9MVMuW6w9/PiTIkeb0nm6AOgk9dvdeg4XILkj +djewSSwq0YdWgJuIhYkNE0/guN0LGZtVvFyTQlECfy3l4m4VwlaPRSSYUxUzULs7 +xc34lL1mf09pWxWebZw4ILQQ90DGWOSD/Zgq6CryRfYgsYqXmGNgDbUFRTWh2DMq +6IoLG3wiqrKSW2oFQL3UOzws0ag7C+6aqKnydpQoEtaP5X+DfAWdAOqnsOP1Ry+W +VTrmtVm4yLiMPBnsw3I= +-----END PRIVATE KEY----- diff --git a/dev/rstuf/keys/root/root1 b/dev/rstuf/keys/root/root1 new file mode 100644 index 000000000000..d7b5a7f7300a --- /dev/null +++ b/dev/rstuf/keys/root/root1 @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgIP90wnfOXb44C0fH +xubol3Vc2jsIYgBTQq0Oh84d3RWhRANCAARwthJnIUZ4p1Y23l1YVue/o303IcKh +Q0twboZkjEvA3xDwxR0d047EaQOfIFFImkhn+v+gMQJJPB8JiF2iDB4s +-----END PRIVATE KEY----- diff --git a/dev/rstuf/keys/root/root2 b/dev/rstuf/keys/root/root2 new file mode 100644 index 000000000000..d197b99c99a8 --- /dev/null +++ b/dev/rstuf/keys/root/root2 @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIGiI3w9x2HZ9UKGi51USN5JN2wtppaYVCRIBTp8ESaj3 +-----END PRIVATE KEY----- From 75e9a94897773f89059ded5964cdf665dc12984f Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Thu, 22 Feb 2024 19:30:56 +0100 Subject: [PATCH 03/11] Include RSTUF Worker dev config for online key Include in `docker-compose.yml` the configuration for using local online key for development. - Use local volume from `./dev/rstuf/keys/online` as `/keyvault` - Use `/keyvault` as RSTUF Worker setting `RSTUF_ONLINE_KEY_DIR` Signed-off-by: Kairo Araujo --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 77538326b9eb..5d548fe8be79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,8 +173,10 @@ services: image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.14.0b1 volumes: - rstuf-metadata:/var/opt/repository-service-tuf/storage + - ./dev/rstuf/keys/online:/keyvault environment: - RSTUF_STORAGE_BACKEND=LocalStorage + - RSTUF_ONLINE_KEY_DIR=/keyvault - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage - RSTUF_BROKER_SERVER=redis://redis/1 - RSTUF_REDIS_SERVER=redis://redis From 03fdf9b69b17d68de910d54151b64ec749ddc615 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Thu, 22 Feb 2024 20:20:36 +0100 Subject: [PATCH 04/11] Add `repository-service-tuf` (RSTUF CLI) for dev Include the RSTUF command line interface (`repository-service-tuf`). This tool will be used in the development enviroment to perform tasks on the RSTUF service. This tool will be used primary for RSTUF bootstrap. Signed-off-by: Kairo Araujo --- requirements/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index d233587fc74f..b9b34ac968f8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,3 +3,4 @@ hupper>=1.9 pip-tools>=1.0 pyramid_debugtoolbar>=2.5 pip-api +repository-service-tuf \ No newline at end of file From 7880045754cc1c9769b3a2818d138744a0d4da71 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Thu, 22 Feb 2024 21:13:17 +0100 Subject: [PATCH 05/11] Include to dev enviroment RSTUF bootstrap inittuf Include the RSTUF `bootstrap.json` payload in the `dev/rstuf/` folder. The bootstrap payload has the RSTUF complete ceremony process. It uses the keys available in the `dev/rstuf/keys` This commit also includes the `inittuf` in Makefile. The information in the development docs. - context (PEP 458) - command (bootstrap) - use (try out API) Signed-off-by: Kairo Araujo Signed-off-by: Lukas Puehringer --- Makefile | 3 + dev/rstuf/bootstrap.json | 92 ++++++++++++++++++++++++ docs/dev/development/getting-started.rst | 39 ++++++++++ 3 files changed, 134 insertions(+) create mode 100644 dev/rstuf/bootstrap.json diff --git a/Makefile b/Makefile index 1e7893c311fa..a64bad6b0d9e 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,9 @@ resetdb: .state/docker-build-base initdb: .state/docker-build-base .state/db-populated $(MAKE) reindex +inittuf: .state/docker-build-base + docker compose run --rm web rstuf admin ceremony -b -u -f dev/rstuf/bootstrap.json --api-server http://rstuf-api + runmigrations: .state/docker-build-base docker compose run --rm web python -m warehouse db upgrade head diff --git a/dev/rstuf/bootstrap.json b/dev/rstuf/bootstrap.json new file mode 100644 index 000000000000..fd7792424a86 --- /dev/null +++ b/dev/rstuf/bootstrap.json @@ -0,0 +1,92 @@ +{ + "settings": { + "roles": { + "root": { + "expiration": 365 + }, + "targets": { + "expiration": 365 + }, + "snapshot": { + "expiration": 1 + }, + "timestamp": { + "expiration": 1 + }, + "bins": { + "expiration": 1, + "number_of_delegated_bins": 4 + } + } + }, + "metadata": { + "root": { + "signatures": [ + { + "keyid": "c6d8bf2e4f48b41ac2ce8eca21415ca8ef68c133b47fc33df03d4070a7e1e9cc", + "sig": "19dd6b1d5da8149b5a490efc8137beedb85ae036255244b2eba909efe05561636e56c0f9a3fe219601602c142b74cc9d2ab5ba18016cb1f3fb81f16f4cb89100" + } + ], + "signed": { + "_type": "root", + "version": 1, + "spec_version": "1.0.31", + "expires": "2025-02-21T13:58:51Z", + "consistent_snapshot": true, + "keys": { + "50d7e110ad65f3b2dba5c3cfc8c5ca259be9774cc26be3410044ffd4be3aa5f3": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEcLYSZyFGeKdWNt5dWFbnv6N9NyHC\noUNLcG6GZIxLwN8Q8MUdHdOOxGkDnyBRSJpIZ/r/oDECSTwfCYhdogweLA==\n-----END PUBLIC KEY-----\n" + }, + "x-rstuf-key-name": "my ecdsa root key" + }, + "c6d8bf2e4f48b41ac2ce8eca21415ca8ef68c133b47fc33df03d4070a7e1e9cc": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "4f66dabebcf30628963786001984c0b75c175cdcf3bc4855933a2628f0cd0a0f" + }, + "x-rstuf-key-name": "my ed25519 root key" + }, + "2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241": { + "keytype": "rsa", + "scheme": "rsassa-pss-sha256", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwhX6rioiL/cX5Ys32InF\nU52H8tL14QeX0tacZdb+AwcH6nIh97h3RSHvGD7Xy6uaMRmGldAnSVYwJHqoJ5j2\nynVzU/RFpr+6n8Ps0QFg5GmlEqZboFjLbS0bsRQcXXnqJNsVLEPT3ULvu1rFRbWz\nAMFjNtNNk5W/u0GEzXn3D03jIdhD8IKAdrTRf0VMD9TRCXLdMmEU2vkf1NVUnOTb\n/dRX5QA8TtBylVnouZknbavQ0J/pPlHLfxUgsKzodwDlJmbPG9BWwXqQCmP0DgOG\nNIZ1X281MOBaGbkNVEuntNjCSaQxQjfALVVU5NAfal2cwMINtqaoc7Wa+TWvpFEI\nWwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "x-rstuf-online-key-uri": "fn:2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241" + } + }, + "roles": { + "root": { + "keyids": [ + "50d7e110ad65f3b2dba5c3cfc8c5ca259be9774cc26be3410044ffd4be3aa5f3", + "c6d8bf2e4f48b41ac2ce8eca21415ca8ef68c133b47fc33df03d4070a7e1e9cc" + ], + "threshold": 1 + }, + "targets": { + "keyids": [ + "2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241" + ], + "threshold": 1 + }, + "snapshot": { + "keyids": [ + "2f685fa7546f1856b123223ab086b3def14c89d24eef18f49c32508c2f60e241" + ], + "threshold": 1 + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index d6cddaf02582..f82749f9057e 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -242,6 +242,45 @@ or that the ``static`` container has finished compiling the static assets: or maybe something else. +Bootstrapping the TUF Metadata Repository +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable PyPI Index Signing (`PEP 458 `_), +you have to first bootstrap the TUF metadata repository. + +Wait until `make serve` has finished, then run: + +.. code-block:: console + + make inittuf + +You should see the following line at the bottom of the output: + +.. code-block:: console + + Bootstrap completed using `dev/rstuf/bootstrap.json`. 🔐 🎉 + + +This command sends a static *bootstrap payload* to the RSTUF API. The payload +includes the TUF trust root for development and other configuration. + +By calling this API, RSTUF creates the TUF metadata repository, installs the +TUF trust root for development, and creates the initial set of TUF metadata. + +.. note:: + + The RSTUF API is exposed only for development purposes and will not be + available in production. Currently, no upload hooks or automatic metadata + update tasks are configured to interact with RSTUF. + + Take a look at the `RSTUF API documentation + `_ + to see how you can simulate artifact upload or removal, and how they affect + the TUF metadata repository: + + * RSTUF API: http://localhost:8001 + * TUF Metadata Repository: http://localhost:9001/tuf-metadata/ + Resetting the development database ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 55af401349318ac88abac1d856d16c8b236603a3 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Thu, 22 Feb 2024 21:49:40 +0100 Subject: [PATCH 06/11] Expose TUF metadata on files:9001 It exposes the TUF metadata at `http://files:9001:tuf-metadata` in the container network. For developers at `http://localhost:9001/tuf-metadata`. This commit includes the entry on `docker-compose.yml` and the `rstuf-metadata` volume which is attached to the `files` container. Signed-off-by: Kairo Araujo --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5d548fe8be79..ecbbac8c3b18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,6 +141,7 @@ services: - packages-archive:/var/opt/warehouse/packages-archive - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - rstuf-metadata:/var/opt/warehouse/tuf-metadata ports: - "9001:9001" From 3dc8006738f26bfdb375c1b92d99ffc6ca8daa8f Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Tue, 12 Mar 2024 15:33:04 +0100 Subject: [PATCH 07/11] docs: fix formatting for `make serve` mention Fix formatting for `make server` mention in getting-started doc. Signed-off-by: Kairo Araujo --- docs/dev/development/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index f82749f9057e..ee01bedabb83 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -248,7 +248,7 @@ Bootstrapping the TUF Metadata Repository To enable PyPI Index Signing (`PEP 458 `_), you have to first bootstrap the TUF metadata repository. -Wait until `make serve` has finished, then run: +Wait until ``make serve`` has finished, then run: .. code-block:: console From 61c3f760a082f6ae40ec2e5f0e710bb5917c91e3 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Mon, 25 Mar 2024 16:12:58 +0100 Subject: [PATCH 08/11] Set rstuf services to depend on redis Signed-off-by: Kairo Araujo --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ecbbac8c3b18..ad57098c2586 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -169,6 +169,9 @@ services: - RSTUF_REDIS_SERVER=redis://redis - RSTUF_REDIS_SERVER_DB_RESULT=1 - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + depends_on: + redis: + condition: service_started rstuf-worker: image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.14.0b1 @@ -187,6 +190,8 @@ services: depends_on: db: condition: service_healthy + redis: + condition: service_started static: build: From ceff4e70236c2c8a363aeb6a2e6aee0a9f3c5834 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Mon, 25 Mar 2024 16:28:30 +0100 Subject: [PATCH 09/11] fixup! Include to dev enviroment RSTUF bootstrap inittuf --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a64bad6b0d9e..e85d88322ba4 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,9 @@ resetdb: .state/docker-build-base initdb: .state/docker-build-base .state/db-populated $(MAKE) reindex -inittuf: .state/docker-build-base +inittuf: .state/db-migrated + docker compose up -d rstuf-api + docker compose up -d rstuf-worker docker compose run --rm web rstuf admin ceremony -b -u -f dev/rstuf/bootstrap.json --api-server http://rstuf-api runmigrations: .state/docker-build-base From d2ee527b445a2be34cca595cd7719d425c29b02e Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Mon, 25 Mar 2024 19:07:47 +0100 Subject: [PATCH 10/11] fixup! Include to dev enviroment RSTUF bootstrap inittuf Signed-off-by: Kairo Araujo --- docs/dev/development/getting-started.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index ee01bedabb83..f68a780873b2 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -248,8 +248,6 @@ Bootstrapping the TUF Metadata Repository To enable PyPI Index Signing (`PEP 458 `_), you have to first bootstrap the TUF metadata repository. -Wait until ``make serve`` has finished, then run: - .. code-block:: console make inittuf From 5f5282d23ef9e899e88bdc6ea77cea38bcc6bcdd Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Fri, 5 Apr 2024 05:47:55 -0400 Subject: [PATCH 11/11] send SIGKILL to rstuf processes for now --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ad57098c2586..3174b31cdecb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,6 +164,7 @@ services: image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.12.0b1 ports: - 8001:80 + stop_signal: SIGKILL environment: - RSTUF_BROKER_SERVER=redis://redis/1 - RSTUF_REDIS_SERVER=redis://redis @@ -178,6 +179,7 @@ services: volumes: - rstuf-metadata:/var/opt/repository-service-tuf/storage - ./dev/rstuf/keys/online:/keyvault + stop_signal: SIGKILL environment: - RSTUF_STORAGE_BACKEND=LocalStorage - RSTUF_ONLINE_KEY_DIR=/keyvault