From 9ce5b5564910a2023d8448744005d121accd1402 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Tue, 24 Sep 2024 18:33:07 +1000 Subject: [PATCH] Fix example Docker Compose environment Pull Horde's custom postgres image Build aihorde Docker image in two stages to minimise rebuild time Establish minimal necessary configuration to bring up a working composition Fix pre-commit config so black and ruff hooks run Catch and report exceptions in is_redis_up() Add missing final EOL markers to some files (per black) Update documentation --- .pre-commit-config.yaml | 4 +--- CONTRIBUTING.md | 4 +++- Dockerfile | 43 +++++++++++++++++++++++++++++++-------- README_docker.md | 23 ++++++++++++++++----- docker-compose.yaml | 27 +++++++++++++++++------- horde/redis_ctrl.py | 9 +++++++- sql_statements/4.43.0.txt | 2 +- sql_statements/README.md | 2 +- 8 files changed, 87 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bdaca924..b0acc038 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,9 +20,7 @@ repos: rev: v0.4.3 hooks: - id: ruff -repos: - - repo: https://github.com/fsfe/reuse-tool +- repo: https://github.com/fsfe/reuse-tool rev: v4.0.3 hooks: - id: reuse - \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c201a563..68c256ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,8 @@ SPDX-License-Identifier: AGPL-3.0-or-later * start server with `python server.py -vvvvi --horde stable` * You can now connect to http://localhost:7001 +To run the AI Horde with Docker or Docker Compose, see the [README_docker.md](README_docker.md). + # How to contribute to the AI Horde code We are happy you have ideas to improve this service and we welcome all contributors. @@ -85,4 +87,4 @@ Note that the pre-commit will not complain if you forget to add your copyright n # Code of Conduct -We expect all contributors to follow the [Anarchist code of conduct](https://wiki.dbzer0.com/the-anarchist-code-of-conduct/). Do not drive away other contributors due to intended or unintended on bigotry. \ No newline at end of file +We expect all contributors to follow the [Anarchist code of conduct](https://wiki.dbzer0.com/the-anarchist-code-of-conduct/). Do not drive away other contributors due to intended or unintended on bigotry. diff --git a/Dockerfile b/Dockerfile index 87bf9cf3..88ad21fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,52 @@ # SPDX-FileCopyrightText: 2024 Tazlin +# SPDX-FileCopyrightText: 2024 ceruleandeep # # SPDX-License-Identifier: AGPL-3.0-or-later # Use a slim base image for Python 3.10 -FROM python:3.10-slim +FROM python:3.10-slim AS python + + +## +## BUILD STAGE +## +FROM python AS python-build-stage # Install Git RUN apt-get update && apt-get install -y git -# Set the working directory +RUN --mount=type=cache,target=/root/.cache pip install --upgrade pip + +# Build dependencies +COPY ./requirements.txt . +RUN --mount=type=cache,target=/root/.cache \ + pip wheel --wheel-dir /usr/src/app/wheels \ + -r requirements.txt + + +## +## RUN STAGE +## +FROM python AS python-run-stage + +# git is required in the run stage because one dependency is not available in PyPI +RUN apt-get update && apt-get install -y git + +RUN --mount=type=cache,target=/root/.cache pip install --upgrade pip + +# Install dependencies +COPY --from=python-build-stage /usr/src/app/wheels /wheels/ +COPY ./requirements.txt . +RUN pip install --no-cache-dir --no-index --find-links=/wheels/ \ + -r requirements.txt \ + && rm -rf /wheels/ + WORKDIR /app -# Copy the source code to the container COPY . /app -# Install the dependencies -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir --prefer-binary -r requirements.txt - # Set the environment variables -ENV PROFILE= +ENV PROFILE="" # Set the command to run when the container starts CMD ["python", "server.py", "-vvvvi", "--horde", "stable"] diff --git a/README_docker.md b/README_docker.md index 3a823ffc..3a5cfdff 100644 --- a/README_docker.md +++ b/README_docker.md @@ -22,16 +22,29 @@ Run the following command in your project root folder (the folder where your Doc docker build -t aihorde:latest . ``` -## with Docker-compose +## with Docker Compose -Create `.env_docker` file to deliver access information of services used together such as Redis and Postgres. +[docker-compose.yaml](docker-compose.yaml) is provided to run the AI-Horde with Redis and Postgres. -Copy the `.env_template` file in the root folder to create the .env_docker file. +Copy the `.env_template` file in the root folder to create the `.env_docker` file. -[docker-compose.yaml](docker-compose.yaml) Change the file as needed. +```bash +cp .env_template .env_docker +``` + +To use the supplied `.env_template` with the supplied `docker-compose.yaml`, you will need to set: + +```bash +# .env_docker +REDIS_IP="redis" +REDIS_SERVERS='["redis"]' +USE_SQLITE=0 +POSTGRES_URL="postgres" +``` +Then run the following command in your project root folder: ```bash # run in background -docker-compose up -d +docker compose up --build -d ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 36e58264..6f3e1986 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,19 +1,28 @@ # SPDX-FileCopyrightText: 2024 Tazlin +# SPDX-FileCopyrightText: 2024 ceruleandeep # # SPDX-License-Identifier: AGPL-3.0-or-later -version: '3' services: aihorde: + build: + context: . + dockerfile: Dockerfile image: aihorde:latest container_name: aihorde ports: - "7001:7001" # The port number written in front of the colon (:) is the port number to be exposed to the outside, so if you change it, you can access it with localhost:{changePort}. environment: - - PROFILE=docker # If you write a profile, the .env_{PROFILE} file is read. + # Flask obtains its environment variables from the .env file. + # If you set a profile, the .env_{PROFILE} file is read instead. + - PROFILE=docker volumes: - - .env_docker:/app/.env_docker # You can replace the local pre-built .env file with the container's file. + # .env_{PROFILE} is copied into the image when it is built. + # So that you can change the environment variables without rebuilding the image, mount the .env file. + - .env_docker:/app/.env_docker + # Likewise, you can mount the horde directory to change the source code without rebuilding the image. + - ./horde:/app/horde networks: - aihorde_network depends_on: @@ -21,13 +30,14 @@ services: - redis postgres: - image: postgres:15.3-alpine + image: ghcr.io/haidra-org/ai-horde-postgres:latest container_name: postgres restart: always environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: admin - POSTGRES_DB: postgres + POSTGRES_PASSWORD: changeme + volumes: + # Use a named volume to persist the data even if the container is deleted. + - postgres_data:/var/lib/postgresql/data/ ports: - "5432:5432" networks: @@ -45,3 +55,6 @@ services: networks: aihorde_network: driver: bridge + +volumes: + postgres_data: diff --git a/horde/redis_ctrl.py b/horde/redis_ctrl.py index d9f1e5f6..8f6ef023 100644 --- a/horde/redis_ctrl.py +++ b/horde/redis_ctrl.py @@ -24,7 +24,14 @@ def is_redis_up() -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex((redis_hostname, redis_port)) == 0 + try: + return s.connect_ex((redis_hostname, redis_port)) == 0 + except socket.gaierror as e: + # connect_ex suppresses exceptions from POSIX connect() call + # but can still raise gaierror if e.g. the hostname is invalid. + # This may be transient, so log the error and return False. + logger.error(f"Redis server at {redis_hostname}:{redis_port} is not reachable: {e}") + return False def is_local_redis_up() -> bool: diff --git a/sql_statements/4.43.0.txt b/sql_statements/4.43.0.txt index 967b9ebe..9fb608d7 100644 --- a/sql_statements/4.43.0.txt +++ b/sql_statements/4.43.0.txt @@ -10,4 +10,4 @@ CREATE INDEX idx_workers_allow_painting ON public.workers USING btree(allow_pain CREATE INDEX idx_workers_allow_post_processing ON public.workers USING btree(allow_post_processing); CREATE INDEX idx_workers_allow_controlnet ON public.workers USING btree(allow_controlnet); CREATE INDEX idx_workers_allow_sdxl_controlnet ON public.workers USING btree(allow_sdxl_controlnet); -CREATE INDEX idx_workers_allow_lora ON public.workers USING btree(allow_lora); \ No newline at end of file +CREATE INDEX idx_workers_allow_lora ON public.workers USING btree(allow_lora); diff --git a/sql_statements/README.md b/sql_statements/README.md index 04ae9af8..efa8b143 100644 --- a/sql_statements/README.md +++ b/sql_statements/README.md @@ -22,4 +22,4 @@ SPDX-License-Identifier: AGPL-3.0-or-later - `compile_*gen_stats_*.sql` - These files defined stored procedures which populated the `compiled_*` tables and generally represent minute/hour/day/total statistics about generations. - `cron_jobs/` - - Schedules any stats compile jobs via `schedule_cron_job`. \ No newline at end of file + - Schedules any stats compile jobs via `schedule_cron_job`.