diff --git a/.eslintrc.js b/.eslintrc.js index a32108b766..0940d023f6 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -66,7 +66,7 @@ module.exports = { // JavaScript for Node.js { - files: ['src/backend/**/*.js', 'src/tools/**/*.js'], + files: ['src/backend/**/*.js', 'src/tools/**/*.js', 'src/api/**/*.js'], env: { node: true, }, diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000000..f61bbec492 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,65 @@ +# Docker and Telescope + +## Introduction + +Telescope uses Docker to deploy all the different parts of our app. If you haven't +worked with Docker before, it's worth taking a few minutes to [learn how it works](https://docs.docker.com/get-started/). + +You'll see Docker used in a few places + +## Setup + +See the [environment setup doc](environment-setup.md) for info specific to your platform. + +Once installed, Docker uses the following commands: + +- [`docker`](https://docs.docker.com/engine/reference/commandline/cli/) +- [`docker-compose`](https://docs.docker.com/compose/reference/) + +## Running Telescope via Docker + +We have a number of docker-compose files that control all the apps that we ship: + +- `docker-compose.yml` - the development version of our "classic" Telescope app (front-end and back-end) +- `docker-compose-production.yml` - the production version of our "classic" Telescope app (front-end and back-end) + +We also have files for our new Microservices Back-end: + +- `./src/api/docker-compose-api.yml` - the development version +- `./src/api/docker-compose-api-production.yml` - the production version + +The docker-compose files define a set of separate servers and services that can +be run together with a single command. + +``` +# run our development version of the entire Telescope app, building any containers as necessary +docker-compose -f docker-compose.yml up --build + +# stop the running containers +docker-compose -f docker-compose.yml down +``` + +If you want to run a specific app or apps, you can name them: + +``` +# run our development version of the entire Telescope app, building any containers as necessary +docker-compose -f docker-compose.yml up --build login redis telescope +``` + +### Running the Microservices + +For your convenience, you can use the following `npm` scripts: + +``` +# start the microservices containers and gateway in development +npm run api:start + +# stop the containers +npm run api:stop +``` + +The services will now be available via the defined routes: + +| Service | URL | +| ------------------------ | ------------------------------------ | +| Background Image Service | http://image.docker.localhost/image/ | diff --git a/env.example b/env.example index 21ce2a91ae..063dbd15db 100644 --- a/env.example +++ b/env.example @@ -137,3 +137,5 @@ GITHUB_TOKEN= # If we wish to override default collection # UNSPLASH_COLLECTION_ID="" + +IMAGE_PORT=4444 diff --git a/env.production b/env.production index 39674f3f86..63e515d940 100644 --- a/env.production +++ b/env.production @@ -88,3 +88,5 @@ GITHUB_TOKEN= # If we wish to override default collection # UNSPLASH_COLLECTION_ID="" + +IMAGE_PORT=4444 diff --git a/env.staging b/env.staging index c9035e3d99..a6312a3e68 100644 --- a/env.staging +++ b/env.staging @@ -92,3 +92,5 @@ GITHUB_TOKEN= # If we wish to override default collection # UNSPLASH_COLLECTION_ID="" + +IMAGE_PORT=4444 diff --git a/package.json b/package.json index a246cfbc65..31af6956bf 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "@seneca/telescope", + "name": "@senecacdot/telescope", "private": true, "version": "1.6.0", "description": "A tool for tracking blogs in orbit around Seneca's open source involvement", @@ -27,6 +27,9 @@ "html-elements": "./tools/html-elements.js" }, "scripts": { + "api:start": "docker-compose -f ./src/api/docker-compose-api.yml up --build -d", + "api:stop": "docker-compose -f ./src/api/docker-compose-api.yml down", + "install:image-service": "cd src/api/image && npm install", "install:autodeployment": "cd tools/autodeployment && npm install", "install:next": "cd src/frontend/next && npm install", "install:gatsby": "cd src/frontend/gatsby && npm install", @@ -44,7 +47,7 @@ "eslint": "eslint .", "eslint-fix": "eslint --fix .", "lint": "npm run eslint", - "postinstall": "run-s install:gatsby install:next install:autodeployment", + "postinstall": "run-s install:*", "prettier": "prettier --write \"./**/*.{md,jsx,json,html,css,js,yml}\"", "prettier-check": "prettier --check \"./**/*.{md,jsx,json,html,css,js,yml}\"", "pretest": "npm run lint", @@ -138,6 +141,6 @@ "supertest": "4.0.2" }, "engines": { - "node": ">=10.0.0" + "node": ">=12.0.0" } } diff --git a/src/api/config/filebeat.yml b/src/api/config/filebeat.yml new file mode 100644 index 0000000000..ec01d51df1 --- /dev/null +++ b/src/api/config/filebeat.yml @@ -0,0 +1,19 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + +processors: + - add_docker_metadata: ~ + +output.elasticsearch: + hosts: ['http://elasticsearch:9200'] + +setup.kibana: + host: 'http://kibana:5601' + dashboards.enabled: true diff --git a/src/api/docker-compose-api-production.yml b/src/api/docker-compose-api-production.yml new file mode 100644 index 0000000000..943194e521 --- /dev/null +++ b/src/api/docker-compose-api-production.yml @@ -0,0 +1,174 @@ +version: '3' + +services: + # API Gateway + traefik: + image: traefik:v2.4 + container_name: 'traefik' + restart: unless-stopped + command: + - '--api.insecure=true' + - '--providers.docker=true' + - '--providers.docker.exposedbydefault=true' + - '--entrypoints.web.address=:80' + - '--entrypoints.websecure.address=:443' + ports: + - '80:80' + - '443:443' + - '8080:8080' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + + # ELK Stack + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3 + container_name: 'elasticsearch' + restart: unless-stopped + environment: + - bootstrap.memory_lock=true + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + - discovery.type=single-node + # See the following: + # - https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html, + # - https://github.com/deviantony/docker-elk/issues/243 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - '9200' + healthcheck: + interval: 20s + retries: 10 + test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' + labels: + - 'traefik.enable=true' + - 'traefik.http.routers.elastic.rule=Host(`elastic.docker.localhost`)' + - 'traefik.http.routers.elastic.middlewares=es-stripprefix' + - 'traefik.http.middlewares.es-stripprefix.stripprefix.prefixes=/es' + - 'traefik.http.services.elastic.loadbalancer.server.port=9200' + + kibana: + image: docker.elastic.co/kibana/kibana:7.9.3 + container_name: 'kibana' + restart: unless-stopped + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - ELASTICSEARCH_URL=http://elasticsearch:9200 + depends_on: + elasticsearch: + condition: service_healthy + volumes: + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - '5601' + healthcheck: + interval: 10s + retries: 20 + test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status + labels: + - 'traefik.enable=true' + - 'traefik.http.routers.kibana.rule=Host(`kibana.docker.localhost`)' + - 'traefik.backend=kibana' + + # System Metrics Logging + metricbeat: + image: docker.elastic.co/beats/metricbeat:7.9.3 + container_name: 'metricbeat' + restart: unless-stopped + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + depends_on: + elasticsearch: + condition: service_healthy + + # Logging + filebeat: + image: docker.elastic.co/beats/filebeat:7.10.2 + container_name: 'filebeat' + restart: unless-stopped + # Need root for access to Docker daemon at unix:///var/run/docker.sock + user: root + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - KIBANA_HOST=http://kibana:5601 + volumes: + - ./config/filebeat.yml:/usr/share/filebeat/filebeat.yml:rw + # Allows us to report on docker from the hosts information. + - /var/run/docker.sock:/var/run/docker.sock + # Allows us to load container log path as specified in filebeat.yml + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: filebeat -e -strict.perms=false + depends_on: + elasticsearch: + condition: service_healthy + + # Application Performance Monitoring + apm: + image: docker.elastic.co/apm/apm-server:7.10.2 + container_name: 'apm' + restart: unless-stopped + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - KIBANA_HOST=http://kibana:5601 + ports: + - '8200' + healthcheck: + test: + [ + 'CMD', + 'curl', + '--write-out', + "'HTTP %{http_code}'", + '--silent', + '--output', + '/dev/null', + 'http://apm:8200/healthcheck', + ] + retries: 10 + interval: 10s + depends_on: + elasticsearch: + condition: service_healthy + + # Micro Services + image: + container_name: 'image' + restart: unless-stopped + build: + context: ./image + dockerfile: Dockerfile + environment: + - NODE_ENV=production + - IMAGE_PORT=4444 + - SERVICE_NAME=image + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - ELASTIC_APM_SERVER_URL=http://apm:8200 + depends_on: + elasticsearch: + condition: service_healthy + traefik: + condition: service_started + ports: + - '4444' + labels: + # Traefik routing + - 'traefik.http.routers.image.rule=Host(`image.docker.localhost`)' + # Enable gzip compression + - 'traefik.http.routers.image.middlewares=test-compress' + - 'traefik.http.middlewares.test-compress.compress=true' + # ELK Logging + - 'co.elastic.logs/json.keys_under_root: true' + - 'co.elastic.logs/json.overwrite_keys: true' + - 'co.elastic.logs/json.add_error_key: true' + - 'co.elastic.logs/json.expand_keys: true' + - 'co.elastic.logs/json.message_key: message' + +volumes: + elasticsearch: diff --git a/src/api/docker-compose-api.yml b/src/api/docker-compose-api.yml new file mode 100644 index 0000000000..a4fbc4764b --- /dev/null +++ b/src/api/docker-compose-api.yml @@ -0,0 +1,41 @@ +version: '3' + +services: + # API Gateway + traefik: + image: traefik:v2.4 + container_name: 'traefik' + restart: unless-stopped + command: + - '--log.level=DEBUG' + - '--api.insecure=true' + - '--providers.docker=true' + - '--providers.docker.exposedbydefault=true' + - '--entrypoints.web.address=:80' + ports: + - '80:80' + - '8080:8080' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + + # Micro Services + image: + container_name: 'image' + restart: unless-stopped + build: + context: ./image + dockerfile: Dockerfile + environment: + - IMAGE_PORT=4444 + - SERVICE_NAME=image + depends_on: + traefik: + condition: service_started + ports: + - '4444' + labels: + # Traefik routing + - 'traefik.http.routers.image.rule=Host(`image.docker.localhost`)' + # Enable gzip compression + - 'traefik.http.routers.image.middlewares=test-compress' + - 'traefik.http.middlewares.test-compress.compress=true' diff --git a/src/api/image/.dockerignore b/src/api/image/.dockerignore new file mode 100644 index 0000000000..b5d1637224 --- /dev/null +++ b/src/api/image/.dockerignore @@ -0,0 +1,6 @@ +.dockerignore +node_modules +npm-debug.log +Dockerfile +.git +.gitignore diff --git a/src/api/image/.gitignore b/src/api/image/.gitignore new file mode 100644 index 0000000000..33c62ace5b --- /dev/null +++ b/src/api/image/.gitignore @@ -0,0 +1,4 @@ +# Unsplash photos are downloaded on startup +photos/*.jpg +# We have a single photo we use by default until that happens +!photos/default.jpg diff --git a/src/api/image/Dockerfile b/src/api/image/Dockerfile new file mode 100644 index 0000000000..1d6a492d5d --- /dev/null +++ b/src/api/image/Dockerfile @@ -0,0 +1,19 @@ +FROM node:lts-alpine as base + +# https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/ +RUN apk add dumb-init + +# TODO: Add feeding in the port from a .env file + +# Force production env, regardless of what we get from .env +ENV NODE_ENV production + +WORKDIR /app + +COPY --chown=node:node . . + +RUN npm install ci --only=production + +USER node + +CMD ["dumb-init", "node", "server.js"] diff --git a/src/api/image/README.md b/src/api/image/README.md new file mode 100644 index 0000000000..18cc6c51f8 --- /dev/null +++ b/src/api/image/README.md @@ -0,0 +1,47 @@ +# Image Service + +The Image Service provides optimized images for backgrounds. + +## Install + +``` +npm install +``` + +## Usage + +``` +# normal mode +npm start + +# dev mode with automatic restarts +npm run dev +``` + +By default the server is running on http://localhost:4444/. + +You can use any/all the following optional query params: + +1. `t` - the desired image type. Must be one of `jpeg`, `jpg`, `webp`, `png`. Defaults to `jpeg`. +1. `w` - the desired width. Must be between `200` and `2000`. Defaults to `800` if missing. +1. `h` - the desired height. Must be between `200` and `3000`. + +NOTE: if both `w` and `h` are used, the image will be resized/cropped to cover those dimensions + +### Examples + +- `GET /image` - returns the default background JPEG with width = 800px +- `GET /image?w=1024`- returns the default background JPEG with width = 1024px +- `GET /image?h=1024`- returns the default background JPEG with height = 1024px +- `GET /image?w=1024&h=1024`- returns the default background JPEG with width = 1024px and height = 1024px +- `GET /image?t=png`- returns the default background JPEG with width = 800px as a PNG + +- `GET /gallery` - an HTML gallery page will be returned of all backgrounds +- `GET /image/default.jpg` - returns a specific image (e.g., `default.jpg` or any other) from the available backgrounds + +- `GET /healthcheck` - returns `{ "status": "ok" }` if everything is running properly + +## Docker + +- To build and tag: `docker build . -t telescope_img_svc:latest` +- To run locally: `docker run -p 4444:4444 telescope_img_svc:latest` diff --git a/src/api/image/bin/unsplash-download.js b/src/api/image/bin/unsplash-download.js new file mode 100755 index 0000000000..7e141aa518 --- /dev/null +++ b/src/api/image/bin/unsplash-download.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +const { logger } = require('@senecacdot/satellite'); +const stream = require('stream'); +const path = require('path'); +const fs = require('fs'); +const got = require('got'); +const { promisify } = require('util'); + +const unsplashPhotoUrls = require('../unsplash-photos.json'); +const { photosDir } = require('../src/lib/photos'); + +const pipeline = promisify(stream.pipeline); + +/** + * Download the image at `url` and write to `filename` + * @param {string} url - the URL to the image + * @param {string} filename - the local filename to use when writing + */ +function downloadPhoto(url, filename) { + return pipeline(got.stream(url), fs.createWriteStream(filename)).then(() => + logger.debug(`Wrote ${url} to ${filename}`) + ); +} + +/** + * Check if the given filename is already available locally + * @param {string} filename - filename for image + * @returns {boolean} + */ +async function shouldDownload(filename) { + try { + await fs.promises.access(filename, fs.FS_OK); + // File already exists, skip + return false; + } catch (err) { + // No such file, we need it + return true; + } +} + +/** + * Download all Unsplash.com images to photos/ + * @returns {Promise} + */ +function downloadUnsplashPhotos() { + // The width we request from Unsplash. This will be our max width + const defaultPhotoWidth = 2000; + + // Process this list, since we only need to download photos we don't already have locally + const downloads = unsplashPhotoUrls + .map((url) => { + const unsplashId = url.trim().replace('https://unsplash.com/photos/', ''); + return { + filename: path.join(photosDir, `${unsplashId}.jpg`), + url: `https://unsplash.com/photos/${unsplashId}/download?force=true&w=${defaultPhotoWidth}`, + }; + }) + .filter((photo) => shouldDownload(photo.filename)); + + return Promise.all(downloads.map(({ url, filename }) => downloadPhoto(url, filename))); +} + +downloadUnsplashPhotos() + .then(() => { + logger.info(`Finished downloading Unsplash photos to ${photosDir}`); + return process.exit(); + }) + .catch((err) => { + logger.warn(`Couldn't download Unsplash photos`, err.message); + return process.exit(1); + }); diff --git a/src/api/image/package.json b/src/api/image/package.json new file mode 100644 index 0000000000..abd1617259 --- /dev/null +++ b/src/api/image/package.json @@ -0,0 +1,32 @@ +{ + "name": "@senecacdot/image-service", + "private": true, + "version": "1.0.0", + "description": "A service for optimizing images", + "scripts": { + "dev": "nodemon server.js | pino-pretty -c -t", + "start": "node server.js", + "clean": "del \"./photos/*.jpg\" \"!./photos/default.jpg\"" + }, + "repository": "Seneca-CDOT/telescope", + "license": "BSD-2-Clause", + "bugs": { + "url": "https://github.com/Seneca-CDOT/telescope/issues" + }, + "homepage": "https://github.com/Seneca-CDOT/telescope#readme", + "dependencies": { + "@senecacdot/satellite": "^1.3.0", + "celebrate": "^13.0.4", + "got": "^11.8.1", + "http-errors": "^1.8.0", + "sharp": "^0.27.1" + }, + "engines": { + "node": ">=12.0.0" + }, + "devDependencies": { + "del-cli": "^3.0.1", + "nodemon": "^2.0.7", + "pino-pretty": "^4.5.0" + } +} diff --git a/src/api/image/photos/default.jpg b/src/api/image/photos/default.jpg new file mode 100755 index 0000000000..655f152a15 Binary files /dev/null and b/src/api/image/photos/default.jpg differ diff --git a/src/api/image/server.js b/src/api/image/server.js new file mode 100644 index 0000000000..511a7827b2 --- /dev/null +++ b/src/api/image/server.js @@ -0,0 +1,14 @@ +const { Satellite } = require('@senecacdot/satellite'); + +const image = require('./src/routes/image'); +const gallery = require('./src/routes/gallery'); +const { download } = require('./src/lib/photos'); + +const service = new Satellite(); + +service.router.use('/image', image); +service.router.use('/gallery', gallery); + +const port = parseInt(process.env.IMAGE_PORT || 4444, 10); +// Once the server is running, start downloading any missing Unsplash photos +service.start(port, download); diff --git a/src/api/image/src/lib/image.js b/src/api/image/src/lib/image.js new file mode 100644 index 0000000000..f7b375dc14 --- /dev/null +++ b/src/api/image/src/lib/image.js @@ -0,0 +1,37 @@ +const sharp = require('sharp'); + +// https://sharp.pixelplumbing.com/api-resize +const resize = (width, height) => sharp({ failOnError: false }).resize({ width, height }); + +/** + * Optimizes (and maybe resizes) the image stream. If a type is given, + * use that, or default to JPEG. If a width or height are given, use those, + * otherwise fit the image to the width x height using CSS cover sizing. + */ +function optimize({ imgStream, width, height, imageType, res }) { + // Picks the appropriate image type and set the content type. + let transformer; + switch (imageType) { + case 'webp': + res.type('image/webp'); + // https://sharp.pixelplumbing.com/api-output#webp + transformer = () => resize(width, height).webp(); + break; + case 'png': + res.type('image/png'); + // https://sharp.pixelplumbing.com/api-output#avif + transformer = () => resize(width, height).png(); + break; + case 'jpeg': + case 'jpg': + default: + res.type('image/jpeg'); + // https://sharp.pixelplumbing.com/api-output#jpeg + transformer = () => resize(width, height).jpeg(); + break; + } + + return imgStream.pipe(transformer()); +} + +module.exports.optimize = optimize; diff --git a/src/api/image/src/lib/photos.js b/src/api/image/src/lib/photos.js new file mode 100644 index 0000000000..c86b4a92d6 --- /dev/null +++ b/src/api/image/src/lib/photos.js @@ -0,0 +1,49 @@ +const { logger } = require('@senecacdot/satellite'); +const { execFile } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +// We put all downloaded photos in the photos/ dir +const photosDir = path.join(__dirname, '../..', 'photos'); + +// By default we only have 1 photo available until we've downloaded the rest +let photos = ['default.jpg']; + +// Download the Unsplash photos in the background, and update our list when done +function download() { + logger.info('Started downloading Unsplash photos...'); + const filename = path.join(__dirname, '../..', '/bin/unsplash-download.js'); + execFile(filename, (error, stdout, stderr) => { + if (error) { + logger.warn({ stderr }, 'Unable to download Unsplash photos'); + } else { + fs.readdir(photosDir, (err, photoFilenames) => { + if (err) { + logger.error({ err }, 'Unable to read downloaded photos'); + return; + } + photos = photoFilenames.filter((photoFilename) => photoFilename.endsWith('.jpg')); + logger.info('Finished downloading Unsplash photos.'); + }); + } + }); +} + +// Get a random image filename from the photos/ directory. Prior to download() +// being called, this will just be the default.jpg image. Afterward it will +// be any of the Unsplash photos defined in unsplash-photos.json. +function getRandomPhotoFilename() { + const photoFilename = photos[Math.floor(Math.random() * photos.length)]; + return path.join(photosDir, photoFilename); +} + +// Get a specific image filename from the photos/ directory. +function getPhotoFilename(image) { + return path.join(photosDir, image); +} + +exports.download = download; +exports.getRandomPhotoFilename = getRandomPhotoFilename; +exports.getPhotoFilename = getPhotoFilename; +exports.photosDir = photosDir; +exports.getPhotos = () => [...photos]; diff --git a/src/api/image/src/routes/gallery.js b/src/api/image/src/routes/gallery.js new file mode 100644 index 0000000000..107fe583d7 --- /dev/null +++ b/src/api/image/src/routes/gallery.js @@ -0,0 +1,43 @@ +const { Router } = require('@senecacdot/satellite'); +const { getPhotos } = require('../lib/photos'); + +const beginning = ` + +
+ +