From ea178f9aff0a72b9438fce888294a8b0e5139832 Mon Sep 17 00:00:00 2001 From: Parag Ekbote Date: Fri, 13 Dec 2024 16:01:50 +0000 Subject: [PATCH 1/5] update makefiles and .yml files for docker --- e2e/Makefile | 1 + tools/docker-compose-dataset-viewer.yml | 3 +++ tools/docker-compose-dev-dataset-viewer.yml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/e2e/Makefile b/e2e/Makefile index 655ec0b738..a4f2c0a3ad 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -3,6 +3,7 @@ export ADMIN_UVICORN_NUM_WORKERS := 2 export ADMIN_UVICORN_PORT := 9081 export API_HF_JWT_ALGORITHM := EdDSA export API_HF_JWT_PUBLIC_KEY_URL := https://hub-ci.huggingface.co/api/keys/jwt +export API_HF_JWT_ADDITIONAL_PUBLIC_KEYS := # ^ it's not tested in the e2e tests, but it's good to ensure we're able to fetch it at least. export ADMIN_HF_TIMEOUT_SECONDS := 10 export API_HF_TIMEOUT_SECONDS := 10 diff --git a/tools/docker-compose-dataset-viewer.yml b/tools/docker-compose-dataset-viewer.yml index eb690a49f8..db4fdd9f23 100644 --- a/tools/docker-compose-dataset-viewer.yml +++ b/tools/docker-compose-dataset-viewer.yml @@ -45,6 +45,9 @@ services: ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS-0.2} ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH-/api/whoami-v2} ADMIN_MAX_AGE: ${ADMIN_MAX_AGE-10} + CACHED_ASSETS_BASE_URL: + ASSETS_BASE_URL: + DEV_NETWORK_MODE: ${DEV_NETWORK_MODE} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} # uvicorn diff --git a/tools/docker-compose-dev-dataset-viewer.yml b/tools/docker-compose-dev-dataset-viewer.yml index 05d686d66c..cb8438ebff 100644 --- a/tools/docker-compose-dev-dataset-viewer.yml +++ b/tools/docker-compose-dev-dataset-viewer.yml @@ -47,6 +47,11 @@ services: ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS-10.0} ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH-/api/whoami-v2} ADMIN_MAX_AGE: ${ADMIN_MAX_AGE-10} + API_HF_JWT_PUBLIC_KEY_URL: ${API_HF_JWT_PUBLIC_KEY_URL} + API_HF_JWT_ADDITIONAL_PUBLIC_KEYS: ${API_HF_JWT_ADDITIONAL_PUBLIC_KEYS} + CACHED_ASSETS_BASE_URL: + ASSETS_BASE_URL: + DEV_NETWORK_MODE: ${DEV_NETWORK_MODE} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} # uvicorn From 81e87218d7837d6c53414dece9a7cfa78356dad4 Mon Sep 17 00:00:00 2001 From: Parag Ekbote Date: Sat, 11 Jan 2025 17:48:17 +0000 Subject: [PATCH 2/5] Add environment variables in makefiles. --- tools/Docker.mk | 3 +++ tools/docker-compose-dataset-viewer.yml | 3 --- tools/docker-compose-dev-dataset-viewer.yml | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/Docker.mk b/tools/Docker.mk index 92d881ee3f..06385cc9fd 100644 --- a/tools/Docker.mk +++ b/tools/Docker.mk @@ -1,3 +1,6 @@ +export API_HF_JWT_PUBLIC_KEY_URL := https://hub-ci.huggingface.co/api/keys/jwt +export API_HF_JWT_ADDITIONAL_PUBLIC_KEYS := + .PHONY: down down: docker compose -f $(DOCKER_COMPOSE) down --remove-orphans --volumes diff --git a/tools/docker-compose-dataset-viewer.yml b/tools/docker-compose-dataset-viewer.yml index db4fdd9f23..eb690a49f8 100644 --- a/tools/docker-compose-dataset-viewer.yml +++ b/tools/docker-compose-dataset-viewer.yml @@ -45,9 +45,6 @@ services: ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS-0.2} ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH-/api/whoami-v2} ADMIN_MAX_AGE: ${ADMIN_MAX_AGE-10} - CACHED_ASSETS_BASE_URL: - ASSETS_BASE_URL: - DEV_NETWORK_MODE: ${DEV_NETWORK_MODE} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} # uvicorn diff --git a/tools/docker-compose-dev-dataset-viewer.yml b/tools/docker-compose-dev-dataset-viewer.yml index cb8438ebff..af0e633840 100644 --- a/tools/docker-compose-dev-dataset-viewer.yml +++ b/tools/docker-compose-dev-dataset-viewer.yml @@ -47,10 +47,6 @@ services: ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS-10.0} ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH-/api/whoami-v2} ADMIN_MAX_AGE: ${ADMIN_MAX_AGE-10} - API_HF_JWT_PUBLIC_KEY_URL: ${API_HF_JWT_PUBLIC_KEY_URL} - API_HF_JWT_ADDITIONAL_PUBLIC_KEYS: ${API_HF_JWT_ADDITIONAL_PUBLIC_KEYS} - CACHED_ASSETS_BASE_URL: - ASSETS_BASE_URL: DEV_NETWORK_MODE: ${DEV_NETWORK_MODE} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} From ebc5fe2cba9771a7ddda4eee5bf560058ac689ac Mon Sep 17 00:00:00 2001 From: Parag Ekbote Date: Sat, 11 Jan 2025 17:52:56 +0000 Subject: [PATCH 3/5] Remove unused variable --- tools/docker-compose-dev-dataset-viewer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/docker-compose-dev-dataset-viewer.yml b/tools/docker-compose-dev-dataset-viewer.yml index af0e633840..05d686d66c 100644 --- a/tools/docker-compose-dev-dataset-viewer.yml +++ b/tools/docker-compose-dev-dataset-viewer.yml @@ -47,7 +47,6 @@ services: ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS-10.0} ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH-/api/whoami-v2} ADMIN_MAX_AGE: ${ADMIN_MAX_AGE-10} - DEV_NETWORK_MODE: ${DEV_NETWORK_MODE} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} # uvicorn From 20d7a266764b4297cf4a3eb4dcdbbdbddd0166bf Mon Sep 17 00:00:00 2001 From: Parag Ekbote Date: Mon, 13 Jan 2025 13:37:07 +0000 Subject: [PATCH 4/5] Move variables to Makefile. --- Makefile | 3 +++ tools/Docker.mk | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bfa50dec0b..9a51bdba3d 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ export PORT_SSE_API := 8185 export PORT_WORKER := 8186 export PORT_WEBHOOK := 8187 export PORT_REVERSE_PROXY := 8100 +export API_HF_JWT_PUBLIC_KEY_URL := https://hub-ci.huggingface.co/api/keys/jwt +export API_HF_JWT_ADDITIONAL_PUBLIC_KEYS := # environment variables per target start: export COMPOSE_PROJECT_NAME := datasets-server @@ -15,6 +17,7 @@ stop: export COMPOSE_PROJECT_NAME := datasets-server dev-start: export COMPOSE_PROJECT_NAME := dev-datasets-server dev-stop: export COMPOSE_PROJECT_NAME := dev-datasets-server + # makefile variables per target start: DOCKER_COMPOSE := ./tools/docker-compose-dataset-viewer.yml stop: DOCKER_COMPOSE := ./tools/docker-compose-dataset-viewer.yml diff --git a/tools/Docker.mk b/tools/Docker.mk index 06385cc9fd..1959c785a3 100644 --- a/tools/Docker.mk +++ b/tools/Docker.mk @@ -1,5 +1,4 @@ -export API_HF_JWT_PUBLIC_KEY_URL := https://hub-ci.huggingface.co/api/keys/jwt -export API_HF_JWT_ADDITIONAL_PUBLIC_KEYS := + .PHONY: down down: From d1dade0de26c30eecfb7ca9412c58d4ac5fbaacf Mon Sep 17 00:00:00 2001 From: Parag Ekbote Date: Mon, 13 Jan 2025 14:00:55 +0000 Subject: [PATCH 5/5] Remove spacing. --- Makefile | 1 - tools/Docker.mk | 2 -- 2 files changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 9a51bdba3d..39325fd7db 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ stop: export COMPOSE_PROJECT_NAME := datasets-server dev-start: export COMPOSE_PROJECT_NAME := dev-datasets-server dev-stop: export COMPOSE_PROJECT_NAME := dev-datasets-server - # makefile variables per target start: DOCKER_COMPOSE := ./tools/docker-compose-dataset-viewer.yml stop: DOCKER_COMPOSE := ./tools/docker-compose-dataset-viewer.yml diff --git a/tools/Docker.mk b/tools/Docker.mk index 1959c785a3..92d881ee3f 100644 --- a/tools/Docker.mk +++ b/tools/Docker.mk @@ -1,5 +1,3 @@ - - .PHONY: down down: docker compose -f $(DOCKER_COMPOSE) down --remove-orphans --volumes