Merge pull request #156 from replicatedhq/emosbaugh/sc-112142/ttl-sh-…

…cost-increase-needs-to-be-addressed

feat: run blob reaper every 72h

.github/workflows/main.yaml

@@ -11,7 +11,7 @@ jobs:
  env:
  HEROKU_API_KEY: ${{secrets.HEROKU_API_KEY}}
  steps:
- - name: Checkout
+ - name: checkout
  uses: actions/checkout@v2
 
  - name: heroku registry login
@@ -52,3 +52,12 @@ jobs:
 
  - name: release reaper
  run: heroku container:release -a ttlsh-hooks reap
+
+ - name: build blob reaper
+ run: docker build -f registry/Dockerfile.blob-reap -t registry.heroku.com/ttlsh-hooks/blob-reap registry
+
+ - name: push blob reaper
+ run: docker push registry.heroku.com/ttlsh-hooks/blob-reap
+
+ - name: release blob reaper
+ run: heroku container:release -a ttlsh-hooks blob-reap

kustomize/overlays/dev/registry.yaml

@@ -16,19 +16,13 @@ spec:
  spec:
  containers:
  - name: ttl-registry
- image: registry:2
- command: ["/entrypoint.sh"]
+ image: ttl-registry
+ command: ["registry", "serve", "/etc/docker/registry/config.yml"]
  ports:
  - containerPort: 5000
  volumeMounts:
  - name: registry-data
  mountPath: /var/lib/registry
- - name: registry-scripts
- mountPath: /entrypoint.sh
- subPath: entrypoint.sh
- - name: registry-scripts
- mountPath: /garbage-collect.sh
- subPath: garbage-collect.sh
  - name: registry-config
  mountPath: /etc/docker/registry/config.yml
  subPath: config.yml
@@ -42,40 +36,21 @@ spec:
  path: /
  port: 5000
  scheme: HTTP
+ - name: ttl-blob-reap
+ image: ttl-blob-reap
+ command: ["/garbage-collect.sh"]
+ volumeMounts:
+ - name: registry-data
+ mountPath: /var/lib/registry
+ - name: registry-config
+ mountPath: /etc/docker/registry/config.yml
+ subPath: config.yml
  volumes:
  - name: registry-data
  emptyDir: {}
  - name: registry-config
  configMap:
  name: ttl-registry-config
- - name: registry-scripts
- configMap:
- name: ttl-registry-scripts
- defaultMode: 0755
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: ttl-registry-scripts
- labels:
- app: ttl-registry
-data:
- entrypoint.sh: |
- #!/bin/sh
- set -e
- # /garbage-collect.sh &
- registry serve /etc/docker/registry/config.yml
- garbage-collect.sh: |
- #!/bin/sh
- while true; do
- sleep 1m
- if [ ! -d "/var/lib/registry/docker" ]; then
- echo "No registry data found, skipping garbage collection"
- else
- echo "Starting garbage collection..."
- registry garbage-collect /etc/docker/registry/config.yml || true
- fi
- done
 ---
 apiVersion: v1
 kind: ConfigMap

okteto.yml

@@ -1,4 +1,10 @@
 build:
+ ttl-registry:
+ context: ./registry
+ dockerfile: ./registry/Dockerfile
+ ttl-blob-reap:
+ context: ./registry
+ dockerfile: ./registry/Dockerfile.blob-reap
  ttl-hooks:
  context: ./hooks
  dockerfile: ./hooks/Dockerfile.hooks
@@ -7,6 +13,8 @@ build:
  dockerfile: ./hooks/Dockerfile.reap
 
 deploy:
+ - cd kustomize/overlays/dev && kustomize edit set image ttl-registry=${OKTETO_BUILD_TTL_REGISTRY_IMAGE}
+ - cd kustomize/overlays/dev && kustomize edit set image ttl-blob-reap=${OKTETO_BUILD_TTL_BLOB_REAP_IMAGE}
  - cd kustomize/overlays/dev && kustomize edit set image ttl-hooks=${OKTETO_BUILD_TTL_HOOKS_IMAGE}
  - cd kustomize/overlays/dev && kustomize edit set image ttl-reaper=${OKTETO_BUILD_TTL_REAPER_IMAGE}
 

registry/Dockerfile

@@ -1,8 +1,11 @@
 FROM registry:2.7.1
 
 ADD ./entrypoint.sh /entrypoint.sh
-ADD ./config.yml /etc/docker/registry/config.yml
+ADD ./config.yml /etc/docker/registry/config.tmpl.yml
 ADD ./garbage-collect.sh /garbage-collect.sh
 
+# entrypoint.sh will copy config.tmpl.yml to config.yml and replace variables
+RUN rm -f /etc/docker/registry/config.yml
+
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["/etc/docker/registry/config.yml"]
+CMD ["registry", "serve", "/etc/docker/registry/config.yml"]

registry/Dockerfile.blob-reap

@@ -0,0 +1,12 @@
+# built from https://github.com/sgalsaleh/distribution/tree/custom-garbage-collect
+FROM replicated/registry-blob-reap:latest
+
+ADD ./entrypoint.sh /entrypoint.sh
+ADD ./config.yml /etc/docker/registry/config.tmpl.yml
+ADD ./garbage-collect.sh /garbage-collect.sh
+
+# entrypoint.sh will copy config.tmpl.yml to config.yml and replace variables
+RUN rm -f /etc/docker/registry/config.yml
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/garbage-collect.sh"]

registry/entrypoint.sh

@@ -2,25 +2,21 @@
 
 set -e
 
-sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml
-sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml
-sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml
-sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml
-sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml
+# in dev we mount this file into the container, so we can ignore the template
+if [ ! -f /etc/docker/registry/config.yml ]; then
+ cp /etc/docker/registry/config.tmpl.yml /etc/docker/registry/config.yml
+ sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml
+ sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml
+ sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml
+ sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml
+ sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml
+fi
 
-if [[ -z "${GCS_KEY_ENCODED}" ]]; then
+if [ -z "${GCS_KEY_ENCODED}" ]; then
  echo "Set GCS_KEY_ENCODED variable"
 else
- echo ${GCS_KEY_ENCODED} | base64 -d > /etc/gcs.json
+ echo "${GCS_KEY_ENCODED}" | base64 -d > /etc/gcs.json
  chmod 0400 /etc/gcs.json
 fi
 
-# Run garbage collection job in background
-# /garbage-collect.sh &
-
-case "$1" in
- *.yaml|*.yml) set -- registry serve "$@" ;;
- serve|garbage-collect|help|-*) set -- registry "$@" ;;
-esac
-
 exec "$@"

registry/garbage-collect.sh

@@ -1,7 +1,10 @@
 #!/bin/sh
 
+echo "Script started"
+
 while true; do
- sleep 1m
  echo "Starting garbage collection..."
  registry garbage-collect /etc/docker/registry/config.yml || true
+ echo "Garbage collection finished"
+ sleep 72h # every 3 days
 done