From eaf28c61b5e931abcd3a06984647a9500982ee64 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Wed, 13 Jan 2021 00:38:26 +0100 Subject: [PATCH 01/13] [FEATURE] Added mobile agent checking using regex --- .env.example | 1 + Makefile | 3 +++ seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 13f8ef7..eeed436 100755 --- a/.env.example +++ b/.env.example @@ -6,6 +6,7 @@ RENDERTRON_CACHE_RESOURCE_URL=http://rendertron:3000/render RENDERTRON_CACHE_RESOURCE_METHOD=GET RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST= RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST=Set-Cookie,Content-Encoding,Transfer-Encoding +RENDERTRON_MOBILE_REGEX=.*Mobile.* API_URL=http://dashboard:80/docs/ API_VER=v1 diff --git a/Makefile b/Makefile index 83f457d..852bdd7 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev up: docker-compose ${DC_CONFIGS} -f docker-compose.yml up +down: + docker-compose ${DC_CONFIGS} -f docker-compose.yml down + daemon: docker-compose ${DC_CONFIGS} -f docker-compose.yml up -d diff --git a/seosnap-cacheserver b/seosnap-cacheserver index 6bc6cdf..115e496 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit 6bc6cdff99617b3c2374c9936166e2f1e24f9725 +Subproject commit 115e4965958f721604de2dda4de25b4b4e1806da diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index 7f931b9..aca0d2a 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit 7f931b9d2b962b219a074dbf612fb4ced356f14d +Subproject commit aca0d2ab4dbd85dc67a74781406983e6f5816c05 From 40e434f757e9a5626e067741210a3b588ab418ed Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:20:36 +0100 Subject: [PATCH 02/13] [FEATURE][N-1] Added minimal release script --- .gitignore | 4 +- Makefile | 9 ++ dev/scripts/release_config.py | 75 ++++++++++ dev/scripts/requirements.txt | 1 + docker-compose.yml | 2 - release/docker-compose.yml | 256 +++++++++++++++++++++++++++++++++ release/install.sh | 24 ++++ release/rendertron-config.json | 3 + seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 11 files changed, 374 insertions(+), 6 deletions(-) create mode 100644 dev/scripts/release_config.py create mode 100644 dev/scripts/requirements.txt create mode 100644 release/docker-compose.yml create mode 100644 release/install.sh create mode 100644 release/rendertron-config.json diff --git a/.gitignore b/.gitignore index e2bd815..24b0214 100755 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,6 @@ dmypy.json # Pyre type checker .pyre/ - +/release/.env +/release/.env.example +/release/*.zip \ No newline at end of file diff --git a/Makefile b/Makefile index 852bdd7..803bf4c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ #!make +.PHONY: all release + DC_CONFIGS = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.yml -f seosnap-cachewarmer/docker-compose.yml -f seosnap-dashboard/docker-compose.yml DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev.yml -f seosnap-cachewarmer/docker-compose.dev.yml -f seosnap-dashboard/docker-compose.dev.yml @@ -37,3 +39,10 @@ develop: git submodule foreach --recursive git fetch origin develop git submodule foreach --recursive git checkout develop echo "Everything is now up to date" + + +release: + python dev/scripts/release_config.py --configs docker-compose.yml seosnap-cacheserver/docker-compose.yml seosnap-cachewarmer/docker-compose.yml seosnap-dashboard/docker-compose.yml docker-compose.yml + rm -rf release/cache release/logs + rm release/release.zip + cd release && zip release.zip * .env.example \ No newline at end of file diff --git a/dev/scripts/release_config.py b/dev/scripts/release_config.py new file mode 100644 index 0000000..aa46f9b --- /dev/null +++ b/dev/scripts/release_config.py @@ -0,0 +1,75 @@ +import yaml +from compose import config +from compose.config.config import ConfigDetails, ConfigFile +from compose.config.serialize import serialize_config + +import os +import argparse +from shutil import copyfile + +parser = argparse.ArgumentParser(description='Seosnap Release Script') +parser.add_argument('--configs', type=str, nargs='+', help='Docker compose config files') +parser.add_argument('--tag', default='latest', help='Tag for the release containers') +args = parser.parse_args() + +CONFIGS = args.configs +TAG = args.tag +OUTPUT_DIR = 'release' +WORKDIR = os.path.abspath(os.curdir) + +REPLACE_MOUNTS = { + './seosnap-cacheserver/rendertron-config.json': './rendertron-config.json' +} + +configs = [] +for file in CONFIGS: + print(f'Reading file: {file}') + with open(file, 'r') as f: + configs.append(ConfigFile(None, yaml.safe_load(f.read()))) + +print('Building config') +env = config.environment.Environment() +details = ConfigDetails( + WORKDIR, + configs, + env +) +cfg = config.load( + details, + False +) + + +def relativize(path: str) -> str: + result = f'./{os.path.relpath(path, WORKDIR)}' if path and path.startswith(WORKDIR) else path + if result in REPLACE_MOUNTS: + result = REPLACE_MOUNTS[result] + return result + + +print('Preprocessing config') +for service in cfg.services: + print(f'\tPreprocessing service: {service["name"]}') + if 'build' in service: + service.pop('build') + + for i, volume in enumerate(service['volumes']): + service['volumes'][i] = volume._replace( + internal=relativize(volume.internal), + external=relativize(volume.external) + ) + + +print(f'Writing composer file') +with open(os.path.join(OUTPUT_DIR, 'docker-compose.yml'), 'w') as f: + f.write(serialize_config(cfg, None, False)) + +copyfile( + os.path.join(WORKDIR, './seosnap-cacheserver/rendertron-config.json'), + os.path.join(OUTPUT_DIR, './rendertron-config.json'), +) + +copyfile( + os.path.join(WORKDIR, './.env.example'), + os.path.join(OUTPUT_DIR, './.env.example'), +) \ No newline at end of file diff --git a/dev/scripts/requirements.txt b/dev/scripts/requirements.txt new file mode 100644 index 0000000..0c67c28 --- /dev/null +++ b/dev/scripts/requirements.txt @@ -0,0 +1 @@ +docker-compose \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 99fa3c2..2abdef7 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,6 @@ services: db: container_name: seosnap_stack_db env_file: ./.env - volumes: - - ./dev/instance/mariadb:/var/lib/mysql networks: - seosnap diff --git a/release/docker-compose.yml b/release/docker-compose.yml new file mode 100644 index 0000000..8080643 --- /dev/null +++ b/release/docker-compose.yml @@ -0,0 +1,256 @@ +networks: + rendertron_net: + driver: bridge + seosnap: + driver: bridge + seosnap_cachewarmer: + driver: bridge + seosnap_dashboard: + driver: bridge +services: + cacheserver: + container_name: seosnap_stack_cacheserver + depends_on: + rendertron: + condition: service_started + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: null + RENDERTRON_CACHE_FILE_SUFFIX: null + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: null + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: null + RENDERTRON_CACHE_LOCK_TIMEOUT: null + RENDERTRON_CACHE_RESOURCE_METHOD: null + RENDERTRON_CACHE_RESOURCE_URL: null + RENDERTRON_CACHE_ROOT: null + RENDERTRON_MOBILE_REGEX: null + TAG: latest + image: experius/seosnap-cacheserver:${TAG} + networks: + rendertron_net: {} + seosnap: {} + ports: + - published: 5000 + target: 5000 + restart: always + volumes: + - ./cache:/app/cache:rw + - ./logs:/app/logs:rw + cachewarmer: + container_name: seosnap_stack_cachewarmer + depends_on: + cacheserver: + condition: service_started + dashboard: + condition: service_started + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: null + API_PASS: null + API_URL: null + API_VER: null + CACHEWARMER_BUFFER_SIZE: null + CACHEWARMER_CACHE_SERVER_URL: null + CACHEWARMER_CONCURRENT_REQUESTS: null + CACHEWARMER_LOG_LEVEL: null + CACHEWARMER_THREADS: null + CACHEWARMER_USER_AGENT: null + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: experius/seosnap-cachewarmer:${TAG} + networks: + seosnap: {} + seosnap_cachewarmer: {} + restart: "no" + volumes: + - ./logs:/code/logs:rw + dashboard: + container_name: seosnap_stack_dashboard + depends_on: + db: + condition: service_started + environment: + ADMIN_DEBUG: null + ADMIN_EMAIL: null + ADMIN_LOG_LEVEL: null + ADMIN_NAME: null + ADMIN_PASS: null + ADMIN_SECRET: null + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: null + DB_HOST_OVERRIDE: db + DB_NAME: null + DB_PASS: null + DB_ROOT_HOST: null + DB_USER: null + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: experius/seosnap-dashboard:${TAG} + networks: + seosnap: {} + seosnap_dashboard: {} + ports: + - published: 80 + target: 80 + restart: unless-stopped + volumes: + - ./logs:/code/logs:rw + db: + container_name: seosnap_stack_db + entrypoint: + - /entrypoint.sh + - --default-authentication-plugin=mysql_native_password + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + MYSQL_DATABASE: ${DB_NAME} + MYSQL_PASSWORD: ${DB_PASS} + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_ROOT_HOST: ${DB_ROOT_HOST} + MYSQL_USER: ${DB_USER} + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: mysql + networks: + seosnap: {} + seosnap_dashboard: {} + ports: + - published: 3306 + target: 3306 + restart: unless-stopped + volumes: + - seosnap_dashboard_db:/var/lib/mysql:rw + rendertron: + container_name: snapstack_rendertron + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: egordm/rendertron:latest + networks: + rendertron_net: {} + seosnap: {} + ports: + - published: 3000 + target: 3000 + restart: always + volumes: + - ./rendertron-config.json:/app/config.json:rw +version: '3.7' +volumes: + seosnap_dashboard_db: {} diff --git a/release/install.sh b/release/install.sh new file mode 100644 index 0000000..49d5fbf --- /dev/null +++ b/release/install.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +if [ "$1" == "clean" ]; then + echo 'Cleaning installation' + sudo rm -rf ./cache +fi + +mkdir -p cache +mkdir -p logs +if [ ! -f ".env" ]; then + cp .env.example .env + echo 'Generating new secret key' + export NEW_SECRET="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c50)"; + sed -i "s/https\:\/\/miniwebtool.com\/django-secret-key-generator\//${NEW_SECRET}/g" .env; + + echo 'Setting new admin user login' + read -p 'Username [snaptron]: ' ADMIN_NAME + read -p 'Email [snaptron@snap.tron]: ' ADMIN_EMAIL + read -sp 'Password [Sn@ptron1337]: ' ADMIN_PASS + sed -i "s/snaptron$/${ADMIN_NAME:-snaptron}/g" .env; + sed -i "s/snaptron@snap.tron/${ADMIN_EMAIL:-snaptron@snap.tron}/g" .env; + sed -i "s/Sn@ptron1337/${ADMIN_PASS:-Sn@ptron1337}/g" .env; +fi + diff --git a/release/rendertron-config.json b/release/rendertron-config.json new file mode 100644 index 0000000..c6b2956 --- /dev/null +++ b/release/rendertron-config.json @@ -0,0 +1,3 @@ +{ + "restrictedUrlPattern": ".*(\\.png|\\.jpg|\\.jpeg|\\.gif|\\.webp|\\.mp4)($|\\?)" +} diff --git a/seosnap-cacheserver b/seosnap-cacheserver index 115e496..d42e300 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit 115e4965958f721604de2dda4de25b4b4e1806da +Subproject commit d42e30072ce29fe2b3b4505233744cd5cfa233f0 diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index aca0d2a..f31dc57 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit aca0d2ab4dbd85dc67a74781406983e6f5816c05 +Subproject commit f31dc571724c037229abe18fa7bdd009af4a0eeb diff --git a/seosnap-dashboard b/seosnap-dashboard index b7830d7..d1e810a 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit b7830d7a99fcfc48a072a5ce3acb81065b01f0da +Subproject commit d1e810a635fd5e64e7cf4bc5af164f4ed2d83d29 From 33c0745b71477478d026f35f2310294222ef8f34 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:32:30 +0100 Subject: [PATCH 03/13] [FEATURE][N-1] Added minor fix regarding tag env var --- .env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.example b/.env.example index eeed436..43d68af 100755 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ +TAG=latest + RENDERTRON_CACHE_DEBUG=0 RENDERTRON_CACHE_LOCK_TIMEOUT=1 RENDERTRON_CACHE_ROOT=./cache From ee191720b3eeec5242ef0d89a4cf7ca8d585c17b Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:45:19 +0100 Subject: [PATCH 04/13] [FEATURE][N-1] Trigger github action --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e85fcb2..3207f10 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,4 @@ make warm A="cache 1,2 use_queue=true" # Clean the queue for websites 1 and 2 make warm A="clean 1,2" ``` +. \ No newline at end of file From 7dcb56b31da0c93b36f51faba4960be3b7cf9f25 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:48:56 +0100 Subject: [PATCH 05/13] [FEATURE][N-1] Trigger github action --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7f64d5..2ef5b7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,8 @@ name: Release on: push: - branches: [ develop ] + branches: + - develop jobs: build: From 737a24d0e3d2ab904f8b6e488c97651e247b6c74 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:53:09 +0100 Subject: [PATCH 06/13] [FEATURE][N-1] Updated release script --- dev/scripts/release_config.py | 6 ++++++ release/docker-compose.yml | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/dev/scripts/release_config.py b/dev/scripts/release_config.py index aa46f9b..a41a379 100644 --- a/dev/scripts/release_config.py +++ b/dev/scripts/release_config.py @@ -21,6 +21,12 @@ './seosnap-cacheserver/rendertron-config.json': './rendertron-config.json' } +if not os.path.exists(os.path.join(WORKDIR, '.env')): + copyfile( + os.path.join(WORKDIR, './.env.example'), + os.path.join(WORKDIR, './.env'), + ) + configs = [] for file in CONFIGS: print(f'Reading file: {file}') diff --git a/release/docker-compose.yml b/release/docker-compose.yml index 8080643..ffadff5 100644 --- a/release/docker-compose.yml +++ b/release/docker-compose.yml @@ -19,7 +19,7 @@ services: ADMIN_LOG_LEVEL: ERROR ADMIN_NAME: snaptron ADMIN_PASS: Sn@ptron1337 - ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + ADMIN_SECRET: https://miniwebtool.com/django-secret-key-generator/ API_NAME: snaptron API_PASS: Sn@ptron1337 API_URL: http://dashboard:80/docs/ @@ -70,7 +70,7 @@ services: ADMIN_LOG_LEVEL: ERROR ADMIN_NAME: snaptron ADMIN_PASS: Sn@ptron1337 - ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + ADMIN_SECRET: https://miniwebtool.com/django-secret-key-generator/ API_NAME: null API_PASS: null API_URL: null @@ -164,7 +164,7 @@ services: ADMIN_LOG_LEVEL: ERROR ADMIN_NAME: snaptron ADMIN_PASS: Sn@ptron1337 - ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + ADMIN_SECRET: https://miniwebtool.com/django-secret-key-generator/ API_NAME: snaptron API_PASS: Sn@ptron1337 API_URL: http://dashboard:80/docs/ @@ -214,7 +214,7 @@ services: ADMIN_LOG_LEVEL: ERROR ADMIN_NAME: snaptron ADMIN_PASS: Sn@ptron1337 - ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + ADMIN_SECRET: https://miniwebtool.com/django-secret-key-generator/ API_NAME: snaptron API_PASS: Sn@ptron1337 API_URL: http://dashboard:80/docs/ From 392226962731aa3730fe31f2403928944407cf4b Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 14:54:56 +0100 Subject: [PATCH 07/13] [FEATURE][N-1] Updated release script --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 803bf4c..610a151 100644 --- a/Makefile +++ b/Makefile @@ -44,5 +44,5 @@ develop: release: python dev/scripts/release_config.py --configs docker-compose.yml seosnap-cacheserver/docker-compose.yml seosnap-cachewarmer/docker-compose.yml seosnap-dashboard/docker-compose.yml docker-compose.yml rm -rf release/cache release/logs - rm release/release.zip + rm -f release/release.zip cd release && zip release.zip * .env.example \ No newline at end of file From c2d4dca4fc9fc4dbf3f42d5e59bb95e9e6face3a Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 15:09:46 +0100 Subject: [PATCH 08/13] [FEATURE][N-1] Updated release script --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ef5b7a..c90fc11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,3 +20,11 @@ jobs: run: pip install -r dev/scripts/requirements.txt - name: Run release script run: make release + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: true + title: "Development Build" + files: | + release/release.zip From 6e66007fdb61e3656ba42c9132d2844aee30fbb7 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 15:34:03 +0100 Subject: [PATCH 09/13] [FEATURE][N-1] Updated release script --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3207f10..9b70478 100644 --- a/README.md +++ b/README.md @@ -85,5 +85,4 @@ make warm A="cache 1,2 use_queue=true" # Clean the queue for websites 1 and 2 make warm A="clean 1,2" -``` -. \ No newline at end of file +``` \ No newline at end of file From 5134ec74d9330d99249962eeebc6a5919ae01338 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 29 Jan 2021 15:36:29 +0100 Subject: [PATCH 10/13] Update release.yml --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c90fc11..9b9f84f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,10 +21,10 @@ jobs: - name: Run release script run: make release - uses: "marvinpinto/action-automatic-releases@latest" - with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "latest" - prerelease: true - title: "Development Build" - files: | - release/release.zip + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: true + title: "Development Build" + files: | + release/release.zip From d417c707c933085174a797fbedd8cd3bdb7dfacc Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sun, 14 Feb 2021 21:33:21 +0100 Subject: [PATCH 11/13] [FEATURE] Added email settings to .env example --- .env.example | 7 +++++++ seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 43d68af..205366f 100755 --- a/.env.example +++ b/.env.example @@ -36,3 +36,10 @@ ADMIN_PASS=Sn@ptron1337 ADMIN_SECRET="https://miniwebtool.com/django-secret-key-generator/" ADMIN_DEBUG=1 ADMIN_LOG_LEVEL=ERROR + +EMAIL_HOST=localhost +EMAIL_HOST_USER= +EMAIL_HOST_PASSWORD= +EMAIL_PORT=25 +EMAIL_USE_TLS=0 +EMAIL_USE_SSL=0 \ No newline at end of file diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index f31dc57..8d06625 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit f31dc571724c037229abe18fa7bdd009af4a0eeb +Subproject commit 8d06625c849b82ba995ef9ad989d26f0be876e7e diff --git a/seosnap-dashboard b/seosnap-dashboard index d1e810a..1a0eea7 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit d1e810a635fd5e64e7cf4bc5af164f4ed2d83d29 +Subproject commit 1a0eea7054a492385fc28bf1c7bad2d43fb1c4d7 From 781654a4a4be3967f0823b430888359c3b40ed78 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sat, 20 Feb 2021 00:33:52 +0100 Subject: [PATCH 12/13] [OTHER] Update submodules --- seosnap-dashboard | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seosnap-dashboard b/seosnap-dashboard index 1a0eea7..4269f23 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit 1a0eea7054a492385fc28bf1c7bad2d43fb1c4d7 +Subproject commit 4269f23843900ebadd9b7ed1d5c23eeb02c9ecc8 From 73acc1df641826cdb11dbdaa837f3be68a00f639 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Tue, 23 Feb 2021 21:08:27 +0100 Subject: [PATCH 13/13] [TASK] Updated readme and dependencies --- .github/workflows/release.yml | 1 + README.md | 55 +++++++++-------------------------- seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 5 files changed, 17 insertions(+), 45 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b9f84f..8ef8a77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,7 @@ name: Release on: push: branches: + - master - develop jobs: diff --git a/README.md b/README.md index 9b70478..f2da7fd 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Setup for the whole seosnap stack including dashboard, cache server and cache wa page caching PWA's. # Installation +## For usage +* Download and Extract release.zip from + * [Releases](https://github.com/experius/SeoSnap/releases) + * [Development Build](https://github.com/experius/SeoSnap/releases/tag/latest) +* Run `./install.sh` or create a .env file manually +* Start the Seosnap stack with `docker-compose up` + +## For development ``` # Clone git clone --recursive git@github.com:experius/SeoSnap.git @@ -16,7 +24,6 @@ make up # Usage * Dashboard: http://127.0.0.1:8080/ (default login: snaptron/Sn@ptron1337) * API Docs: http://127.0.0.1:8080/docs -* PHPMyAdmin: http://127.0.0.1:8081/ * Cache Server: http://127.0.0.1:5000/render/\ Logs directory ./logs @@ -39,50 +46,14 @@ Check the nginx.conf in the example folder ![diagram](https://github.com/experius/SeoSnap/raw/master/assets/diagram.png) -### Dashboard +### [Dashboard](https://github.com/experius/SeoSnap-Dashboard) In the dashboard you add the website url along with the website sitemap that you want to make 'SeoSnaps' off. -### Crawler +### [Crawler / Cache Warmer](https://github.com/experius/SeoSnap-Cache-Warmer) When the crawler is started it connects with the dashboard api. It uses scrapy to crawl the sitemap. The scrapy results are send to the administration/dashboard. Scrapy requests are send to the cache server. In a similar way that you would do a request to rendertron. -### Cache Server +### [Cache Server](https://github.com/experius/SeoSnap-Cache-Server) The cache server is a simple file caching server. If a file exist with the content of the page it serves the html from the file. If not, it renders the requested url with rendertron and saves the html output in a file. To refresh the cache the cache-warmer uses PUT requests instead of GET. This will force update from the cache file. -# Build with -![diagram](https://github.com/experius/SeoSnap/raw/master/assets/software.png) - -## Usage cache warmer [See](https://github.com/experius/SeoSnap-Cache-Warmer/blob/master/README.md) -### Commands -#### Cache -Handles caching of pages associated to given website -``` -Usage: crawl.py cache [OPTIONS] WEBSITE_IDS - -Options: - --follow_next BOOLEAN Follows rel-next links if enabled - --recache BOOLEAN Recached all pages instead of not yet cached ones - --use_queue BOOLEAN Cache urls from the queue instead of the sitemap - --load BOOLEAN Whether already loaded urls should be scraped instead - --help Show this message and exit. -``` - -#### Clean -Handles cleaning of the dashboard queue -``` -Usage: crawl.py clean [OPTIONS] WEBSITE_IDS - -Options: - --help Show this message and exit. -``` - -### Examples -``` -# Cache the sitemap of website 1 -make warm A="cache 1" - -# Cache requests in queue for websites 1 and 2 -make warm A="cache 1,2 use_queue=true" - -# Clean the queue for websites 1 and 2 -make warm A="clean 1,2" -``` \ No newline at end of file +# Built with +![diagram](https://github.com/experius/SeoSnap/raw/master/assets/software.png) \ No newline at end of file diff --git a/seosnap-cacheserver b/seosnap-cacheserver index d42e300..55ede16 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit d42e30072ce29fe2b3b4505233744cd5cfa233f0 +Subproject commit 55ede16cd0a0e1c45699874e8b3bc1082bb63b3a diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index 8d06625..1cb2f98 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit 8d06625c849b82ba995ef9ad989d26f0be876e7e +Subproject commit 1cb2f9827021960899a0ad58ddce2537525f0d19 diff --git a/seosnap-dashboard b/seosnap-dashboard index 4269f23..bebe63e 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit 4269f23843900ebadd9b7ed1d5c23eeb02c9ecc8 +Subproject commit bebe63ecd8305dd11f2725efb50fbefd14d75e31