✨ individual compose files

juftin · Apr 5, 2024 · a404561 · a404561
1 parent f5fe1a1
commit a404561
Show file tree

Hide file tree

Showing 33 changed files with 130 additions and 173 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,19 +1,4 @@
 # Secrets
 .env*
 secrets/
-
-# Application Persistence
 appdata/
-
-# Python Files
-__pycache__/
-*.py[cod]
-*$py.class
-.venv/
-site/
-
-# Mac Files
-.DS_Store
-
-# IDE Files
-.idea/
diff --git a/README.md b/README.md
@@ -36,12 +36,9 @@ This repository is a large [docker compose](https://docs.docker.com/compose/)
 project that allows you to deploy a variety of services to your homelab.
 
 At the root of this repository is a `docker-compose.yaml` file that defines
-the entire homelab project - collections of services are broken out into their
-own subdirectories inside of the `stacks` directory. Under each stack,
-every service gets its own subdirectory and a singular `docker-compose.yaml` file
-(`stacks/media-center/plex/docker-compose.yaml`). Using the `include` directive in the
-docker compose files, we create a single stack from the top level that deploys
-everything.
+the entire homelab project - it uses the `include` directive to pull in
+individual service docker compose files from the `stacks` directory.
+Ultimately a single docker compose stack is created that deploys everything.
 
 ```text
 .
@@ -53,25 +50,20 @@ everything.
 │   └── google_oauth.secret                     # Google OAuth Credentials and Whitelist
 ├── stacks
 │   ├── media-center
-│   │   ├── docker-compose.yaml                 # Media-Center Stack Docker Compose File (Plex, Sonarr, etc.)
-│   │   ├── plex                                # Each individual service has its own subdirectory
-│   │   │   └── docker-compose.yaml             # Each service has its own docker-compose.yaml file
-│   │   └── sonarr
-│   │       └── docker-compose.yaml
+│   │   ├── plex.yaml                           # Each individual service has its own docker compose file
+│   │   ├── radarr.yaml
+│   |   ├── ombi.yaml
+│   │   └── sonarr.yaml
 │   ├── traefik                                 # Traefik Reverse Proxy and OAuth
-│   │   ├── docker-compose.yaml                 # Traefik Stack Docker Compose File (Traefik, OAuth, etc.)
-│   │   ├── oauth                               # OAuth Configuration
-│   │   │   └── docker-compose.yaml
-│   │   └── traefik                             # Traefik Configuration
+│   │   ├── oauth                               # OAuth Service
+│   │   └── traefik                             # Traefik Reverse Proxy
 │   │       ├── docker-compose.yaml             # Traefik Docker Compose File (Traefik Only)
-│   │       └── rules
-│   │           ├── middlewares-chains.yml      # Traefik Middlewares Chains
-│   │           ├── middlewares.yml             # Traefik Middlewares
-│   │           └── tls-opts.yml                # Traefik TLS Options
+│   │       └── rules                           # Traefik Middlewares and Rules
+│   │           ├── middlewares-chains.yml
+│   │           ├── middlewares.yml
+│   │           └── tls-opts.yml
 │   └── miscellaneous                           # Non Media Center Services (pihole, chat-gpt-next-web, etc.)
-│       ├── docker-compose.yaml                 # Miscellaneous Stack Docker Compose File
-│       └── chat-gpt-next-web
-│           └── docker-compose.yaml
+│       └── chat-gpt-next-web.yaml
 └── appdata                                     # Application Data Persistent Volumes
     ├── media-center
     │   ├── plex                                # Each individual service has its own subdirectory

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -2,18 +2,42 @@
 # HOMELAB
 ################################################################################
 
-####################################
-# INCLUDED STACKS
-####################################
-
 include:
-    - stacks/traefik/docker-compose.yaml
-    - stacks/media-center/docker-compose.yaml
-    - stacks/miscellaneous/docker-compose.yaml
+    #############################################################################
+    # TRAEFIK (CORE)
+    #############################################################################
+    - stacks/traefik/traefik/docker-compose.yaml # Traefik (Reverse Proxy)
+    - stacks/traefik/duckdns.yaml # DuckDNS (Dynamic DNS)
+    - stacks/traefik/oauth.yaml # OAuth (Authentication)
+    - stacks/traefik/socket-proxy.yaml # Docker Socket Proxy (Security)
+    #############################################################################
+    # MEDIA CENTER
+    #############################################################################
+    - stacks/media-center/watchtower.yaml #  Watchtower (Container Updating)
+    - stacks/media-center/heimdall.yaml # Heimdall (Landing Page)
+    - stacks/media-center/ombi.yaml # Ombi (Download Requests)
+    - stacks/media-center/plex.yaml # Plex (Media Server)
+    - stacks/media-center/tautulli.yaml # Tautulli (Plex Analytics)
+    - stacks/media-center/sonarr.yaml # Sonarr (TV Show Downloads)
+    - stacks/media-center/radarr.yaml # Radarr (Movie Downloads)
+    - stacks/media-center/prowlarr.yaml # Prowlarr (Indexer)
+    - stacks/media-center/readarr.yaml # Readarr (Ebook Downloads)
+    - stacks/media-center/calibre.yaml # Calibre (Books Management)
+    - stacks/media-center/calibre-web.yaml # Calibre Web-UI
+    - stacks/media-center/transmission.yaml # Transmission (Torrents Behind VPN)
+    - stacks/media-center/nzbget.yaml # NZBGet (Usenet Downloading)
+    - stacks/media-center/portainer.yaml # Portainer (Container Management)
+    - stacks/media-center/sftpgo.yaml # SFTPGo (File Management)
+    #############################################################################
+    # MISCELLANEOUS
+    #############################################################################
+    - stacks/miscellaneous/pihole.yaml # Pi-hole (DNS Ad-Blocking)
+    - stacks/miscellaneous/chat-gpt-next-web.yaml # ChatGPT Next Web
+    - stacks/miscellaneous/chatgpt-in-slack.yaml # ChatGPT Slack Bot
 
-####################################
-# CONFIGURATION
-####################################
+################################################################################
+# NETWORK CONFIGURATION
+################################################################################
 
 networks:
     traefik:

diff --git a/docs/applications/media_center.md b/docs/applications/media_center.md
@@ -1,10 +1,5 @@
 # Media Center Stack
 
-When connecting these applications together, it is important to note that they all share a common
-docker network. This means that when you're trying to connect to a service you can simply use
-a service name as the hostname. For example, if you're trying to connect to the `sonarr` service
-from the `ombi` service, you can simply use `http://sonarr:8989` as the hostname.
-
 ## Table of Contents
 
 -   [plex](#plex)

diff --git a/docs/config.md b/docs/config.md
@@ -18,10 +18,10 @@ correct service. It is the first service that you should start with:
 > Once you have all of the pre-requisites set up, you can use the
 > [up-traefik](cli.md#up-traefik) command to start just the Traefik services.
 
-## Apps
+## App Deployment
 
 Which apps to deploy are defined in the `docker-compose.yaml` files. For example,
-to disable all apps from the `miscellaneous` stack, you would comment out the `include` directive
+To disable specific apps in the `media-center` stack, you would comment out the `include` directive
 in the root `docker-compose.yaml` file.
 
 <details><summary>📄 docker-compose.yaml</summary>
@@ -34,22 +34,11 @@ in the root `docker-compose.yaml` file.
 </p>
 </details>
 
-To disable specific apps in the `media-center` stack, you would comment out the `include` directive
-in the `stacks/media-center/docker-compose.yaml` file.
-
-<details><summary>📄 stacks/media-center/docker-compose.yaml</summary>
-<p>
-
-```yaml
---8<-- "stacks/media-center/docker-compose.yaml"
-```
-
-</p>
-</details>
-
-## App Configuration
+## Infrastructure Configuration
 
-These project makes use of a few configuration files to make it easier to manage:
+All services are configured via a `.env` file at the root of the project and a few secret
+files in the `secrets` directory. These files are used to define settings and credentials
+for all services that are deployed. You can copy the example files to get started:
 
 ```shell
 cp docs/example.env .env
@@ -101,3 +90,14 @@ cp -r docs/example-secrets/ secrets/
 
 </p>
 </details>
+
+## App Configuration
+
+Each app has its own configuration process - see the `Applications` documentation
+for more information about a specific app.
+
+When connecting these applications together, it is important to note that they
+all share a common docker network. This means that when you're trying to connect
+to a service you can simply use a service name as the hostname. For example,
+if you're trying to connect to the sonarr service from the ombi service
+you can simply use http://sonarr:8989 as the hostname.
diff --git a/docs/example.env b/docs/example.env
@@ -13,6 +13,9 @@ PGID="1000" # id -g
 
 # UNIVERSAL_RESTART_POLICY="unless-stopped"
 
+# Uncomment to test with Let's Encrypt (HTTPS) Staging Environment
+# LETS_ENCRYPT_ENV="https://acme-staging-v02.api.letsencrypt.org/directory"
+
 ##################################################################
 # DIRECTORY SETUP
 #

diff --git a/docs/index.md b/docs/index.md
@@ -34,12 +34,9 @@ This repository is a large [docker compose](https://docs.docker.com/compose/)
 project that allows you to deploy a variety of services to your homelab.
 
 At the root of this repository is a `docker-compose.yaml` file that defines
-the entire homelab project - collections of services are broken out into their
-own subdirectories inside of the `stacks` directory. Under each stack,
-every service gets its own subdirectory and a singular `docker-compose.yaml` file
-(`stacks/media-center/plex/docker-compose.yaml`). Using the `include` directive in the
-docker compose files, we create a single stack from the top level that deploys
-everything.
+the entire homelab project - it uses the `include` directive to pull in
+individual service docker compose files from the `stacks` directory.
+Ultimately a single docker compose stack is created that deploys everything.
 
 ```text
 .
@@ -51,25 +48,20 @@ everything.
 │   └── google_oauth.secret                     # Google OAuth Credentials and Whitelist
 ├── stacks
 │   ├── media-center
-│   │   ├── docker-compose.yaml                 # Media-Center Stack Docker Compose File (Plex, Sonarr, etc.)
-│   │   ├── plex                                # Each individual service has its own subdirectory
-│   │   │   └── docker-compose.yaml             # Each service has its own docker-compose.yaml file
-│   │   └── sonarr
-│   │       └── docker-compose.yaml
+│   │   ├── plex.yaml                           # Each individual service has its own docker compose file
+│   │   ├── radarr.yaml
+│   |   ├── ombi.yaml
+│   │   └── sonarr.yaml
 │   ├── traefik                                 # Traefik Reverse Proxy and OAuth
-│   │   ├── docker-compose.yaml                 # Traefik Stack Docker Compose File (Traefik, OAuth, etc.)
-│   │   ├── oauth                               # OAuth Configuration
-│   │   │   └── docker-compose.yaml
-│   │   └── traefik                             # Traefik Configuration
+│   │   ├── oauth                               # OAuth Service
+│   │   └── traefik                             # Traefik Reverse Proxy
 │   │       ├── docker-compose.yaml             # Traefik Docker Compose File (Traefik Only)
-│   │       └── rules
-│   │           ├── middlewares-chains.yml      # Traefik Middlewares Chains
-│   │           ├── middlewares.yml             # Traefik Middlewares
-│   │           └── tls-opts.yml                # Traefik TLS Options
+│   │       └── rules                           # Traefik Middlewares and Rules
+│   │           ├── middlewares-chains.yml
+│   │           ├── middlewares.yml
+│   │           └── tls-opts.yml
 │   └── miscellaneous                           # Non Media Center Services (pihole, chat-gpt-next-web, etc.)
-│       ├── docker-compose.yaml                 # Miscellaneous Stack Docker Compose File
-│       └── chat-gpt-next-web
-│           └── docker-compose.yaml
+│       └── chat-gpt-next-web.yaml
 └── appdata                                     # Application Data Persistent Volumes
     ├── media-center
     │   ├── plex                                # Each individual service has its own subdirectory

diff --git a/docs/traefik.md b/docs/traefik.md
@@ -15,18 +15,25 @@ anywhere in the world.
 > to get started.
 >
 > Once you have all of the pre-requisites set up, you can use the
-> [up-traefik](cli.md#up-traefik) command to start just the Traefik services.
+> [up-traefik](cli.md#up-traefik) command to start just the Traefik services
+> and confirm that everything is working as at https://traefik.yourdomain.com.
+> Initial DNS propagation and certificate generation can take some time, so
+> be patient on the first run.
+
+> [!INFO] "Acknowledgements"
+> This configuration was inspired by, and
+> immensely helped by the article at
+> [smarthomebeginner.com](https://www.smarthomebeginner.com/traefik-docker-compose-guide-2024/).
+> Their massive [home server setup on GitHub](https://github.com/htpcBeginner/docker-traefik)
+> and amazing blog series inspired much of this project.
 
-## Special Thank You
-
-This configuration was inspired by, and
-immensely helped by the article at
-[smarthomebeginner.com](https://smarthomebeginner.com/traefik-2-docker-tutorial).
+## Prerequisites
 
-[Here](https://github.com/htpcBeginner/docker-traefik)
-is their massive home server setup on GitHub which this project is based on.
+### Personal Domain
 
-## Prerequisites
+This guide assumes you have a personal domain name that you can use to
+access your services. You can purchase a domain name from a registrar
+like [Cloudflare](https://www.cloudflare.com/products/registrar/).
 
 ### Port Forwarding
 
@@ -43,7 +50,7 @@ DNS services. SmartHomeBeginner has a great guide on setting up CloudFlare
 
 ### Google OAuth 2.0
 
-The Google Oauth 2.0 configuration can be
+A helpful blog post on the Google Oauth 2.0 configuration can be
 found [here](https://www.smarthomebeginner.com/traefik-forward-auth-google-oauth-2022/).
 Essentially you must create a project in the Google Developer Console to enable
 the Google OAuth 2.0 service. You will share credentials with the `oauth` service
@@ -59,6 +66,15 @@ provide CloudFlare with the DuckDNS subdomain to point to your server.
 
 ### File Configuration
 
+All services are configured via a `.env` file at the root of the project and a few secret
+files in the `secrets` directory. These files are used to define settings and credentials
+for all services that are deployed. You can copy the example files to get started:
+
+```shell
+cp docs/example.env .env
+cp -r docs/example-secrets/ secrets/
+```
+
 #### .env
 
 The `.env` needs to be modified in order for the containers to be build
@@ -81,16 +97,29 @@ fields filled out in the `.env` file:
 ```text
 DUCKDNS_TOKEN=XXXXXX-XXX-XXXXX-XXXXX
 DUCKDNS_SUBDOMAIN=example
-
-GOOGLE_CLIENT_ID=XXXXXXXXXXXXX-XXXXXXXXXXXXXXXXX.apps.googleusercontent.com
-GOOGLE_CLIENT_SECRET=XXXXXXXXXXXXXX
-OAUTH_SECRET=RANDOM_STRING_OF_CHARACTERS
-OAUTH_WHITELIST=example@gmail.com,user_1@gmail.com,user_2@gmail.com
-
 CLOUDFLARE_EMAIL=example@gmail.com
-CLOUDFLARE_API_KEY=XXXXXXXXXXXXX
 ```
 
+And you should also update the secrets files:
+
+=== "secrets/google_oauth.secret"
+
+    ```text
+    --8<-- "docs/example-secrets/google_oauth.secret"
+    ```
+
+=== "secrets/cloudflare_api_key.secret"
+
+    ```text
+    --8<-- "docs/example-secrets/cloudflare_api_key.secret"
+    ```
+
+=== "secrets/admin_password"
+
+    ```text
+    --8<-- "docs/example-secrets/admin_password.secret"
+    ```
+
 #### acme.json
 
 You will need to create an empty `acme.json` file for the
@@ -127,7 +156,7 @@ The below example shows you how to create a new service in the docker compose
 stack and make it accessible via Traefik. In this example, we are creating a
 Jupyter notebook service that can be accessed at `jupyter.example.com`.
 
-=== "stacks/miscellaneous/jupyter/docker-compose.yaml"
+=== "stacks/miscellaneous/jupyter.yaml"
 
     ```yaml
     ####################################
@@ -152,21 +181,6 @@ Jupyter notebook service that can be accessed at `jupyter.example.com`.
                 traefik.http.routers.jupyter-rtr.middlewares: chain-oauth-google@file
     ```
 
-=== "stacks/miscellaneous/docker-compose.yaml"
-
-    ```yaml
-    ################################################################################
-    # DOCKER COMPOSE - MISCELLANEOUS
-    ################################################################################
-
-    ####################################
-    # INCLUDED APPLICATIONS
-    ####################################
-
-    include:
-    - jupyter/docker-compose.yaml # Jupyter Lab
-    ```
-
 === "docker-compose.yaml"
 
     ```yaml

diff --git a/...ia-center/calibre-web/docker-compose.yaml → stacks/media-center/calibre-web.yaml b/...ia-center/calibre-web/docker-compose.yaml → stacks/media-center/calibre-web.yaml
diff --git a/.../media-center/calibre/docker-compose.yaml → stacks/media-center/calibre.yaml b/.../media-center/calibre/docker-compose.yaml → stacks/media-center/calibre.yaml