Merge pull request #3385 from se-jaeger/main

Helm Chart for self-hosting

docker-compose.yml

@@ -24,9 +24,9 @@ services:
     command: '/bin/sh ./packages/db/setup.sh' # Also create a demo user with email: demo@omnivore.app, password: demo_password
     environment:
       - PGPASSWORD=postgres
+      - POSTGRES_USER=postgres
       - PG_HOST=postgres
-      - PG_USER=postgres
-      - PG_PASSWORD=postgres
+      - PG_PASSWORD=app_pass
       - PG_DB=omnivore
     depends_on:
       postgres:

packages/db/setup.sh

@@ -1,13 +1,20 @@
 #!/bin/bash
 
-psql --host $PG_HOST -U $PG_USER -d $PG_DB -c "CREATE USER app_user WITH PASSWORD 'app_pass';"  || true
+psql --host $PG_HOST --username $POSTGRES_USER --command "CREATE DATABASE $PG_DB;" || true
+echo "create $PG_DB database"
+
+psql --host $PG_HOST --username $POSTGRES_USER --command "CREATE USER app_user WITH ENCRYPTED PASSWORD '$PG_PASSWORD';" || true
 echo "created app_user"
-yarn workspace @omnivore/db migrate
-psql --host $PG_HOST -U $PG_USER -d $PG_DB -c "GRANT omnivore_user TO app_user;"  || true
+
+PG_USER=$POSTGRES_USER PG_PASSWORD=$PGPASSWORD yarn workspace @omnivore/db migrate
+
+psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "GRANT omnivore_user TO app_user;" || true
 echo "granted omnivore_user to app_user"
 
 # create demo user with email: demo@omnivore.app, password: demo_password
-USER_ID=$(uuidgen)
-PASSWORD='$2a$10$41G6b1BDUdxNjH1QFPJYDOM29EE0C9nTdjD1FoseuQ8vZU1NWtrh6'
-psql --host $PG_HOST -U $PG_USER -d $PG_DB -c "INSERT INTO omnivore.user (id, source, email, source_user_id, name, password) VALUES ('$USER_ID', 'EMAIL', 'demo@omnivore.app', 'demo@omnivore.app', 'Demo User', '$PASSWORD'); INSERT INTO omnivore.user_profile (user_id, username) VALUES ('$USER_ID', 'demo_user');"
-echo "created demo user with email: demo@omnivore.app, password: demo_password"
+if [ -z "${NO_DEMO_USER}" ]; then
+    USER_ID=$(uuidgen)
+    PASSWORD='$2a$10$41G6b1BDUdxNjH1QFPJYDOM29EE0C9nTdjD1FoseuQ8vZU1NWtrh6'
+    psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "INSERT INTO omnivore.user (id, source, email, source_user_id, name, password) VALUES ('$USER_ID', 'EMAIL', 'demo@omnivore.app', 'demo@omnivore.app', 'Demo User', '$PASSWORD'); INSERT INTO omnivore.user_profile (user_id, username) VALUES ('$USER_ID', 'demo_user');"
+    echo "created demo user with email: demo@omnivore.app, password: demo_password"
+fi

self-hosting/Dockerfile

@@ -11,12 +11,12 @@ COPY tsconfig.json .
 COPY .prettierrc .
 COPY .eslintrc .
 
-COPY /self-hosting/selfhost.sh /app/selfhost.sh
 COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
 COPY /packages/api/package.json ./packages/api/package.json
 COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
 COPY /packages/content-handler/package.json ./packages/content-handler/package.json
 COPY /packages/db/package.json ./packages/db/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
 
 RUN yarn install --pure-lockfile
 
@@ -25,9 +25,11 @@ COPY /packages/api ./packages/api
 COPY /packages/text-to-speech ./packages/text-to-speech
 COPY /packages/content-handler ./packages/content-handler
 COPY /packages/db ./packages/db
+COPY /packages/liqe ./packages/liqe
 
 RUN yarn workspace @omnivore/text-to-speech-handler build
 RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
 RUN yarn workspace @omnivore/api build
 
 RUN rm -rf /app/packages/api/node_modules
@@ -50,14 +52,11 @@ COPY --from=builder /app/node_modules /app/node_modules
 COPY --from=builder /app/package.json /app/package.json
 COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
 COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
 COPY --from=builder /app/packages/db/ /app/packages/db/
-COPY --from=builder /app/selfhost.sh /app/selfhost.sh
 
 RUN apk add --no-cache postgresql-client
-RUN chmod 0755 /app/selfhost.sh
 
 EXPOSE 8080
 
-# Using a custom script to customize imageproxy startup and to pass
-# signatureKey through env variable
-ENTRYPOINT ["/app/selfhost.sh"]
+CMD ["yarn", "workspace", "@omnivore/api", "start"]

self-hosting/build-and-push-images.sh

@@ -0,0 +1,38 @@
+: ${BASE_URL?Please set BASE_URL, i.e. your public webserver URL, e.g. 'https://omnivore.example.com'}
+: ${SERVER_BASE_URL?Please set SERVER_BASE_URL, i.e. your public API URL, e.g. 'https://api.omnivore.example.com'}
+: ${HIGHLIGHTS_BASE_URL?Please set HIGHLIGHTS_BASE_URL, i.e. your public webserver URL, e.g. 'https://omnivore.example.com'}
+: ${DOCKER_HUB_USER?Please set DOCKER_HUB_USER}
+
+: ${APP_ENV:=prod}
+: ${DOCKER_TAG:=$(git rev-parse --short HEAD)}
+
+# TODO: Feat: only build specific images
+
+echo ""
+echo Using the following variables for building and pushing:
+echo "  To build Omnivore:"
+echo "    APP_ENV=$APP_ENV"
+echo "    BASE_URL=$BASE_URL"
+echo "    SERVER_BASE_URL=$SERVER_BASE_URL"
+echo "    HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL"
+echo ""
+echo "  To push to Docker Hub "
+echo "    DOCKER_HUB_USER=$DOCKER_HUB_USER"
+echo "    DOCKER_TAG=$DOCKER_TAG"
+echo ""
+
+read -p "OK? Continue [Y, y]: " yn
+case $yn in
+    [Yy]* ) echo;;
+    * ) exit;;
+esac
+
+docker build -t ${DOCKER_HUB_USER}/omnivore-web:${DOCKER_TAG} --build-arg="APP_ENV=${APP_ENV}" --build-arg="BASE_URL=${BASE_URL}" --build-arg="SERVER_BASE_URL=${SERVER_BASE_URL}" --build-arg="HIGHLIGHTS_BASE_URL=${HIGHLIGHTS_BASE_URL}" -f ../packages/web/Dockerfile ..
+docker build -t ${DOCKER_HUB_USER}/omnivore-content-fetch:${DOCKER_TAG} -f ../packages/content-fetch/Dockerfile ..
+docker build -t ${DOCKER_HUB_USER}/omnivore-api:${DOCKER_TAG} -f Dockerfile ..
+docker build -t ${DOCKER_HUB_USER}/omnivore-migrate:${DOCKER_TAG} -f ../packages/db/Dockerfile ..
+
+docker push ${DOCKER_HUB_USER}/omnivore-web:${DOCKER_TAG}
+docker push ${DOCKER_HUB_USER}/omnivore-content-fetch:${DOCKER_TAG}
+docker push ${DOCKER_HUB_USER}/omnivore-api:${DOCKER_TAG}
+docker push ${DOCKER_HUB_USER}/omnivore-migrate:${DOCKER_TAG}

self-hosting/helm/README.md

@@ -0,0 +1,100 @@
+# Deployment using Helm
+
+Helm chart to self-host Omnivore.
+
+## Notes and General Information
+
+This helm chart uses docker images from [`sejaeger/omnivore-*`](https://hub.docker.com/u/sejaeger). If you want to use the Web-UI or build your own images, checkout `../.build-and-push-images.sh`. You will find some hard-coded environment variables (e.g., `PG_DB` or `PG_USER`), please don't change them! Those are also hard-coded in the code base and changing them will likely cause problems. Please have a look at [the values file](values.yaml) and [change it accordingly](https://github.com/bjw-s/helm-charts/blob/main/charts/library/common/values.yaml) to your setup, especially: postgres hostname, elasticsearch URL, omnivore URL.
+
+Omnivore requires Postgres (+vector extension!) and Elasticsearch to store its information. Please make sure to have them up and running. Using the bitnami Helm charts works perfectly fine. However, for Postgres you need to use a custom built image that contains the vector extension: [See this descriptions](https://github.com/pgvector/pgvector/issues/126#issuecomment-1589203644) for more information or simply use `sejaeger/postgres-vector` from docker hub.
+
+This setup uses a couple of secrets to safely store passwords, tokens and private information. It's your responsibility to generate them and create the following secretes accordingly. 
+
+* omnivore-image-proxy-secret
+    * IMAGE_PROXY_SECRET
+* omnivore-jwt-secret
+    * JWT_SECRET
+* omnivore-sso-jwt-secret
+    * SSO_JWT_SECRET
+* omnivore-pg-password
+    * PG_PASSWORD
+* postgres-admin-user-and-password
+    * PGPASSWORD
+    * POSTGRES_USER
+* elasticsearch-auth-secret
+    * ES_PASSWORD
+* omnivore-content-fetch-verification-token
+    * VERIFICATION_TOKEN
+
+
+## Deployment
+
+```console
+helm repo add bjw-s https://bjw-s.github.io/helm-charts
+helm repo update
+```
+
+In order to deploy the manifest for this example, issue the
+following command:
+
+```console
+helm install omnivore bjw-s/app-template --values values.yaml
+```
+
+This will apply the rendered manifest(s) to your cluster.
+
+
+## RSS Subscriptions
+
+Currently, handling RSS subscriptions are not supported for self-hosted instances. However, you can use this simple tool for this: https://github.com/se-jaeger/omnivore-rss-handler-hack.
+
+Adding the following `controller` and `persistence` information triggers the rss-handler hourly. 
+```yaml
+controllers:
+  rss-handler-hack:
+    type: cronjob
+    cronjob:
+      schedule: "*/60 * * * *"
+      failedJobsHistoryLimit: 1
+      successfulJobsHistoryLimit: 3
+      concurrencyPolicy: Forbid
+    containers:
+      rss-handler-hack:
+        image:
+          repository: sejaeger/omnivore-rss-handler-hack
+          tag: v0.2
+          imagePullPolicy: IfNotPresent
+        env:
+          API_URL: "http://omnivore-omnivore-api:8080/api/graphql"
+          CACHE_FILE: "/home/cache.json"
+          FEEDS_FILE: "/home/feeds.json"
+        envFrom:
+          - secretRef:
+              name: omnivore-api-token
+
+persistence:
+  rss-handler-hack:
+    type: persistentVolumeClaim
+    existingClaim: omnivore-pvc
+    advancedMounts:
+      rss-handler-hack: # controller name
+        rss-handler-hack: # container name
+          - path: /home
+```
+
+`FEEDS_FILE` is used to define the subscriptions:
+```json
+{
+  "blog": "https://blog.example/feed",
+  "another-blog": "https://another-blog.example/rss.xml",
+}
+```
+
+`omnivore-api-token` secret contains a single key `API_TOKEN`, which can be generated using the Omnivore Web-UI.
+
+
+
+## Currently not Implemented
+
+* health checks
+* resource limits

self-hosting/helm/values.yaml

@@ -0,0 +1,107 @@
+controllers:
+  main:
+    enabled: false
+  api:
+    enabled: true
+    type: deployment
+    strategy: Recreate
+    containers:
+      api:
+        image:
+          repository: sejaeger/omnivore-api
+          tag: e44616b01
+          pullPolicy: IfNotPresent
+        env:
+          API_ENV: local
+          PG_HOST: postgres.postgres.svc.cluster.local
+          PG_USER: app_user
+          PG_DB: omnivore
+          PG_PORT: 5432
+          PG_POOL_MAX: 20
+          CLIENT_URL: https://omnivore.example.com # URL any client can use to reach the server
+          GATEWAY_URL: http://omnivore-api:8080/api
+          ES_URL: http://elasticsearch.elasticsearch.svc.cluster.local:9200
+          ES_USERNAME: elastic
+          CONTENT_FETCH_URL: http://omnivore-content-fetch:8080/?token=$(VERIFICATION_TOKEN)
+        envFrom:
+          - secretRef:
+              name: omnivore-image-proxy-secret
+          - secretRef:
+              name: omnivore-jwt-secret
+          - secretRef:
+              name: omnivore-sso-jwt-secret
+          - secretRef:
+              name: omnivore-pg-password
+          - secretRef:
+              name: elasticsearch-auth-secret
+          - secretRef:
+              name: omnivore-content-fetch-verification-token
+    initContainers:
+      migrate:
+        image:
+          repository: sejaeger/omnivore-migrate
+          tag: e44616b01
+          pullPolicy: IfNotPresent
+        command: ["/bin/sh", "-c", "--"]
+        args: ["./packages/db/setup.sh"]
+        env:
+          PG_HOST: postgres.postgres.svc.cluster.local
+          PG_USER: app_user
+          PG_DB: omnivore
+          NO_DEMO_USER: 1
+        envFrom:
+          - secretRef:
+              name: omnivore-pg-password
+          - secretRef:
+              name: postgres-admin-user-and-password
+
+  # NOTE: You need to build this image for your specific setup because NEXT.js burns some environment variables into client code
+  # See: https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables#bundling-environment-variables-for-the-browser
+  # Use the ../build-and-push-images.sh script for your convenience
+  web:
+    enabled: true
+    type: deployment
+    strategy: Recreate
+    containers:
+      web:
+        image:
+          repository: sejaeger/omnivore-web
+          tag: e44616b01
+          pullPolicy: IfNotPresent
+
+  content-fetch:
+    enabled: true
+    type: deployment
+    strategy: Recreate
+    containers:
+      content-fetch:
+        image:
+          repository: sejaeger/omnivore-content-fetch
+          tag: e44616b01
+          pullPolicy: IfNotPresent
+        env:
+          REST_BACKEND_ENDPOINT: http://omnivore-api:8080/api
+        envFrom:
+          - secretRef:
+              name: omnivore-jwt-secret
+          - secretRef:
+              name: omnivore-content-fetch-token
+
+service:
+  main:
+    enabled: false
+  api:
+    controller: api
+    ports:
+      http:
+        port: 8080
+  web:
+    controller: web
+    ports:
+      http:
+        port: 8080
+  content-fetch:
+    controller: content-fetch
+    ports:
+      http:
+        port: 8080