Proof of Concept for Image Microservice

.eslintrc.js

@@ -66,7 +66,7 @@ module.exports = {
 
     // JavaScript for Node.js
     {
-      files: ['src/backend/**/*.js', 'src/tools/**/*.js'],
+      files: ['src/backend/**/*.js', 'src/tools/**/*.js', 'src/api/**/*.js'],
       env: {
         node: true,
       },

docs/docker.md

@@ -0,0 +1,65 @@
+# Docker and Telescope
+
+## Introduction
+
+Telescope uses Docker to deploy all the different parts of our app. If you haven't
+worked with Docker before, it's worth taking a few minutes to [learn how it works](https://docs.docker.com/get-started/).
+
+You'll see Docker used in a few places
+
+## Setup
+
+See the [environment setup doc](environment-setup.md) for info specific to your platform.
+
+Once installed, Docker uses the following commands:
+
+- [`docker`](https://docs.docker.com/engine/reference/commandline/cli/)
+- [`docker-compose`](https://docs.docker.com/compose/reference/)
+
+## Running Telescope via Docker
+
+We have a number of docker-compose files that control all the apps that we ship:
+
+- `docker-compose.yml` - the development version of our "classic" Telescope app (front-end and back-end)
+- `docker-compose-production.yml` - the production version of our "classic" Telescope app (front-end and back-end)
+
+We also have files for our new Microservices Back-end:
+
+- `./src/api/docker-compose-api.yml` - the development version
+- `./src/api/docker-compose-api-production.yml` - the production version
+
+The docker-compose files define a set of separate servers and services that can
+be run together with a single command.
+
+```
+# run our development version of the entire Telescope app, building any containers as necessary
+docker-compose -f docker-compose.yml up --build
+
+# stop the running containers
+docker-compose -f docker-compose.yml down
+```
+
+If you want to run a specific app or apps, you can name them:
+
+```
+# run our development version of the entire Telescope app, building any containers as necessary
+docker-compose -f docker-compose.yml up --build login redis telescope
+```
+
+### Running the Microservices
+
+For your convenience, you can use the following `npm` scripts:
+
+```
+# start the microservices containers and gateway in development
+npm run api:start
+
+# stop the containers
+npm run api:stop
+```
+
+The services will now be available via the defined routes:
+
+| Service                  | URL                                  |
+| ------------------------ | ------------------------------------ |
+| Background Image Service | http://image.docker.localhost/image/ |

env.example

@@ -137,3 +137,5 @@ GITHUB_TOKEN=
 
 # If we wish to override default collection
 # UNSPLASH_COLLECTION_ID=""
+
+IMAGE_PORT=4444

env.production

@@ -88,3 +88,5 @@ GITHUB_TOKEN=
 
 # If we wish to override default collection
 # UNSPLASH_COLLECTION_ID=""
+
+IMAGE_PORT=4444

env.staging

@@ -92,3 +92,5 @@ GITHUB_TOKEN=
 
 # If we wish to override default collection
 # UNSPLASH_COLLECTION_ID=""
+
+IMAGE_PORT=4444

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@seneca/telescope",
+  "name": "@senecacdot/telescope",
   "private": true,
   "version": "1.6.0",
   "description": "A tool for tracking blogs in orbit around Seneca's open source involvement",
@@ -27,6 +27,9 @@
     "html-elements": "./tools/html-elements.js"
   },
   "scripts": {
+    "api:start": "docker-compose -f ./src/api/docker-compose-api.yml up --build -d",
+    "api:stop": "docker-compose -f ./src/api/docker-compose-api.yml down",
+    "install:image-service": "cd src/api/image && npm install",
     "install:autodeployment": "cd tools/autodeployment && npm install",
     "install:next": "cd src/frontend/next && npm install",
     "install:gatsby": "cd src/frontend/gatsby && npm install",
@@ -44,7 +47,7 @@
     "eslint": "eslint .",
     "eslint-fix": "eslint --fix .",
     "lint": "npm run eslint",
-    "postinstall": "run-s install:gatsby install:next install:autodeployment",
+    "postinstall": "run-s install:*",
     "prettier": "prettier --write \"./**/*.{md,jsx,json,html,css,js,yml}\"",
     "prettier-check": "prettier --check \"./**/*.{md,jsx,json,html,css,js,yml}\"",
     "pretest": "npm run lint",
@@ -138,6 +141,6 @@
     "supertest": "4.0.2"
   },
   "engines": {
-    "node": ">=10.0.0"
+    "node": ">=12.0.0"
   }
 }

src/api/config/filebeat.yml

@@ -0,0 +1,19 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+
+processors:
+  - add_docker_metadata: ~
+
+output.elasticsearch:
+  hosts: ['http://elasticsearch:9200']
+
+setup.kibana:
+  host: 'http://kibana:5601'
+  dashboards.enabled: true

src/api/docker-compose-api-production.yml

@@ -0,0 +1,174 @@
+version: '3'
+
+services:
+  # API Gateway
+  traefik:
+    image: traefik:v2.4
+    container_name: 'traefik'
+    restart: unless-stopped
+    command:
+      - '--api.insecure=true'
+      - '--providers.docker=true'
+      - '--providers.docker.exposedbydefault=true'
+      - '--entrypoints.web.address=:80'
+      - '--entrypoints.websecure.address=:443'
+    ports:
+      - '80:80'
+      - '443:443'
+      - '8080:8080'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  # ELK Stack
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
+    container_name: 'elasticsearch'
+    restart: unless-stopped
+    environment:
+      - bootstrap.memory_lock=true
+      - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
+      - discovery.type=single-node
+    # See the following:
+    # - https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html,
+    # - https://github.com/deviantony/docker-elk/issues/243
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - type: volume
+        source: elasticsearch
+        target: /usr/share/elasticsearch/data
+    ports:
+      - '9200'
+    healthcheck:
+      interval: 20s
+      retries: 10
+      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
+    labels:
+      - 'traefik.enable=true'
+      - 'traefik.http.routers.elastic.rule=Host(`elastic.docker.localhost`)'
+      - 'traefik.http.routers.elastic.middlewares=es-stripprefix'
+      - 'traefik.http.middlewares.es-stripprefix.stripprefix.prefixes=/es'
+      - 'traefik.http.services.elastic.loadbalancer.server.port=9200'
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.9.3
+    container_name: 'kibana'
+    restart: unless-stopped
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    volumes:
+      - type: volume
+        source: elasticsearch
+        target: /usr/share/elasticsearch/data
+    ports:
+      - '5601'
+    healthcheck:
+      interval: 10s
+      retries: 20
+      test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
+    labels:
+      - 'traefik.enable=true'
+      - 'traefik.http.routers.kibana.rule=Host(`kibana.docker.localhost`)'
+      - 'traefik.backend=kibana'
+
+  # System Metrics Logging
+  metricbeat:
+    image: docker.elastic.co/beats/metricbeat:7.9.3
+    container_name: 'metricbeat'
+    restart: unless-stopped
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+
+  # Logging
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.10.2
+    container_name: 'filebeat'
+    restart: unless-stopped
+    # Need root for access to Docker daemon at unix:///var/run/docker.sock
+    user: root
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - KIBANA_HOST=http://kibana:5601
+    volumes:
+      - ./config/filebeat.yml:/usr/share/filebeat/filebeat.yml:rw
+      # Allows us to report on docker from the hosts information.
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Allows us to load container log path as specified in filebeat.yml
+      - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: filebeat -e -strict.perms=false
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+
+  # Application Performance Monitoring
+  apm:
+    image: docker.elastic.co/apm/apm-server:7.10.2
+    container_name: 'apm'
+    restart: unless-stopped
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - KIBANA_HOST=http://kibana:5601
+    ports:
+      - '8200'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          'curl',
+          '--write-out',
+          "'HTTP %{http_code}'",
+          '--silent',
+          '--output',
+          '/dev/null',
+          'http://apm:8200/healthcheck',
+        ]
+      retries: 10
+      interval: 10s
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+
+  # Micro Services
+  image:
+    container_name: 'image'
+    restart: unless-stopped
+    build:
+      context: ./image
+      dockerfile: Dockerfile
+    environment:
+      - NODE_ENV=production
+      - IMAGE_PORT=4444
+      - SERVICE_NAME=image
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - ELASTIC_APM_SERVER_URL=http://apm:8200
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      traefik:
+        condition: service_started
+    ports:
+      - '4444'
+    labels:
+      # Traefik routing
+      - 'traefik.http.routers.image.rule=Host(`image.docker.localhost`)'
+      # Enable gzip compression
+      - 'traefik.http.routers.image.middlewares=test-compress'
+      - 'traefik.http.middlewares.test-compress.compress=true'
+      # ELK Logging
+      - 'co.elastic.logs/json.keys_under_root: true'
+      - 'co.elastic.logs/json.overwrite_keys: true'
+      - 'co.elastic.logs/json.add_error_key: true'
+      - 'co.elastic.logs/json.expand_keys: true'
+      - 'co.elastic.logs/json.message_key: message'
+
+volumes:
+  elasticsearch:

src/api/docker-compose-api.yml

@@ -0,0 +1,41 @@
+version: '3'
+
+services:
+  # API Gateway
+  traefik:
+    image: traefik:v2.4
+    container_name: 'traefik'
+    restart: unless-stopped
+    command:
+      - '--log.level=DEBUG'
+      - '--api.insecure=true'
+      - '--providers.docker=true'
+      - '--providers.docker.exposedbydefault=true'
+      - '--entrypoints.web.address=:80'
+    ports:
+      - '80:80'
+      - '8080:8080'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  # Micro Services
+  image:
+    container_name: 'image'
+    restart: unless-stopped
+    build:
+      context: ./image
+      dockerfile: Dockerfile
+    environment:
+      - IMAGE_PORT=4444
+      - SERVICE_NAME=image
+    depends_on:
+      traefik:
+        condition: service_started
+    ports:
+      - '4444'
+    labels:
+      # Traefik routing
+      - 'traefik.http.routers.image.rule=Host(`image.docker.localhost`)'
+      # Enable gzip compression
+      - 'traefik.http.routers.image.middlewares=test-compress'
+      - 'traefik.http.middlewares.test-compress.compress=true'

src/api/image/.dockerignore

@@ -0,0 +1,6 @@
+.dockerignore
+node_modules
+npm-debug.log
+Dockerfile
+.git
+.gitignore

src/api/image/.gitignore

@@ -0,0 +1,4 @@
+# Unsplash photos are downloaded on startup
+photos/*.jpg
+# We have a single photo we use by default until that happens
+!photos/default.jpg

src/api/image/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:lts-alpine as base
+
+# https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/
+RUN apk add dumb-init
+
+# TODO: Add feeding in the port from a .env file
+
+# Force production env, regardless of what we get from .env
+ENV NODE_ENV production
+
+WORKDIR /app
+
+COPY --chown=node:node . .
+
+RUN npm install ci --only=production
+
+USER node
+
+CMD ["dumb-init", "node", "server.js"]