From 235d6797e976ab8c4c975a6ec88b09339ebc67be Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sat, 28 Mar 2020 01:18:39 +0100 Subject: [PATCH 1/5] [FEATURE] Makefile configuration. Added merged docker-compose. Added install script instead of a preconfigured env file --- .env | 35 --------------------------- .env.docker | 1 + .env.example | 6 ++--- .gitignore | 3 ++- Makefile | 33 +++++++++++++++++++++++++ README.md | 20 +++++++-------- docker-compose.dev.yml | 55 ++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 55 ++++++++++++------------------------------ seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 11 files changed, 123 insertions(+), 91 deletions(-) delete mode 100755 .env create mode 100644 .env.docker create mode 100644 Makefile create mode 100755 docker-compose.dev.yml diff --git a/.env b/.env deleted file mode 100755 index 1bbcf61..0000000 --- a/.env +++ /dev/null @@ -1,35 +0,0 @@ -RENDERTRON_CACHE_DEBUG=0 -RENDERTRON_CACHE_LOCK_TIMEOUT=1 -RENDERTRON_CACHE_ROOT=./cache -RENDERTRON_CACHE_FILE_SUFFIX=.json -RENDERTRON_CACHE_RESOURCE_URL=http://rendertron:3000/render -RENDERTRON_CACHE_RESOURCE_METHOD=GET -RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST= -RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST=Set-Cookie,Content-Encoding,Transfer-Encoding - -API_URL=http://dashboard:8080/docs/ -API_VER=v1 -API_NAME=snaptron -API_PASS=Sn@ptron1337 - -CACHEWARMER_CACHE_SERVER_URL=http://cacheserver:5000/render -CACHEWARMER_USER_AGENT=Seosnap -CACHEWARMER_THREADS=2 -CACHEWARMER_CONCURRENT_REQUESTS=2 -CACHEWARMER_LOG_LEVEL=ERROR -CACHEWARMER_BUFFER_SIZE=50 - -EXTERNAL_CACHE_SERVER_URL= - -DB_ROOT_HOST=% -DB_HOST=db -DB_NAME=seosnap_dashboard -DB_USER=snaptron -DB_PASS=snaptron - -ADMIN_EMAIL=snaptron@snaptron.nl -ADMIN_NAME=snaptron -ADMIN_PASS=Sn@ptron1337 -ADMIN_SECRET="981xu2xx(b#ak3%$8jp$xc5=&z+b#66w0hx5!4tmfqnfk=9^u&" -ADMIN_DEBUG=1 -ADMIN_LOG_LEVEL=ERROR \ No newline at end of file diff --git a/.env.docker b/.env.docker new file mode 100644 index 0000000..fbbb4ba --- /dev/null +++ b/.env.docker @@ -0,0 +1 @@ +# A hack. Since docker compose wont override this for some reason diff --git a/.env.example b/.env.example index dad2377..f99d721 100755 --- a/.env.example +++ b/.env.example @@ -24,12 +24,12 @@ EXTERNAL_CACHE_SERVER_URL= DB_ROOT_HOST=% DB_HOST=db DB_NAME=seosnap_dashboard -DB_USER=snaptron -DB_PASS=snaptron +DB_USER=snaptron_db +DB_PASS=snaptron_db ADMIN_EMAIL=snaptron@snaptron.nl ADMIN_NAME=snaptron ADMIN_PASS=Sn@ptron1337 ADMIN_SECRET="https://miniwebtool.com/django-secret-key-generator/" ADMIN_DEBUG=1 -ADMIN_LOG_LEVEL=ERROR \ No newline at end of file +ADMIN_LOG_LEVEL=ERROR diff --git a/.gitignore b/.gitignore index dc3e1db..e2bd815 100755 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,13 @@ /.idea /dev/instance !/dev/instance/.gitkeep -!/.env +/.env !/.env.example /cache !/cache/.gitkeep /logs !/logs/.gitkeep +.env # Created by .ignore support plugin (hsz.mobi) ### Python template diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e6b67d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +#!make +DC_CONFIGS = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.yml -f seosnap-cachewarmer/docker-compose.yml -f seosnap-dashboard/docker-compose.yml +DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev.yml -f seosnap-cachewarmer/docker-compose.dev.yml -f seosnap-dashboard/docker-compose.dev.yml + + +up: + docker-compose ${DC_CONFIGS} -f docker-compose.yml up + +build: + docker-compose ${DC_CONFIGS} -f docker-compose.yml build + +dc: + docker-compose ${DC_CONFIGS} -f docker-compose.yml ${A} + +warm: + docker-compose ${DC_CONFIGS} -f docker-compose.yml run cachewarmer ${A} + +install: + echo 'Running installation' + cp .env.example .env + sh seosnap-dashboard/dev/commands/install.sh + +update: + git submodule update --init --recursive + git submodule foreach --recursive git fetch origin master + git submodule foreach --recursive git checkout master + echo "Everything is now up to date" + +develop: + git submodule update --init --recursive + git submodule foreach --recursive git fetch origin develop + git submodule foreach --recursive git checkout develop + echo "Everything is now up to date" diff --git a/README.md b/README.md index a481b55..e85fcb2 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,13 @@ Setup for the whole seosnap stack including dashboard, cache server and cache wa page caching PWA's. # Installation -* Pull the repo (*note: the pull is recursive*) ``` +# Clone git clone --recursive git@github.com:experius/SeoSnap.git -``` -* **IMPORTANT** Update .env file admin username and password. (These have a value default value) -* Start, build and stop the container -``` -docker-compose up --build -d && docker-compose down +# Configure +make install +# Start server +make up ``` # Usage @@ -27,7 +26,8 @@ Cache directory ./cache ## Run cache warmer Make sure you have created a website via dashboard http://127.0.0.1:8080/seosnap/website/add/ ``` -docker-compose run cachewarmer cache +make cachewarmer +make warm A="cache " ``` ## Nginx @@ -78,11 +78,11 @@ Options: ### Examples ``` # Cache the sitemap of website 1 -docker-compose run cachewarmer cache 1 +make warm A="cache 1" # Cache requests in queue for websites 1 and 2 - dc run cachewarmer cache 1,2 use_queue=true +make warm A="cache 1,2 use_queue=true" # Clean the queue for websites 1 and 2 -docker-compose run cachewarmer clean 1,2 +make warm A="clean 1,2" ``` diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100755 index 0000000..303577d --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,55 @@ +version: '3.7' +services: + dashboard: + build: + context: seosnap-dashboard + container_name: seosnap_stack_dashboard + env_file: .env + volumes: + - ./seosnap-dashboard:/code + - ./logs:/code/logs + networks: + - seosnap + + cachewarmer: + build: + context: seosnap-cachewarmer + container_name: seosnap_stack_cachewarmer + env_file: .env + volumes: + - ./seosnap-cachewarmer:/code + - ./logs:/code/logs + depends_on: + - dashboard + - cacheserver + networks: + - seosnap + + cacheserver: + build: + context: seosnap-cacheserver + container_name: seosnap_stack_cacheserver + env_file: .env + volumes: + - ./cache:/app/cache + - ./logs:/app/logs + depends_on: + - rendertron + networks: + - seosnap + + rendertron: + container_name: snapstack_rendertron + networks: + - seosnap + + db: + container_name: seosnap_stack_db + volumes: + - ./dev/instance/mariadb:/var/lib/mysql + networks: + - seosnap + +networks: + seosnap: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index e7f778a..21f46dc 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,22 @@ version: '3.7' services: dashboard: - build: ./seosnap-dashboard - image: seosnap-dashboard - container_name: seosnap_dashboard - env_file: .env - environment: - DB_HOST_OVERRIDE: db + build: + context: seosnap-dashboard + container_name: seosnap_stack_dashboard + env_file: ./.env volumes: - - ./seosnap-dashboard:/code - ./logs:/code/logs - restart: "unless-stopped" - ports: - - 8080:8080 - depends_on: - - db networks: - seosnap cachewarmer: - build: ./seosnap-cachewarmer - image: seosnap-cachewarmer - container_name: seosnap_cachewarmer - env_file: .env + build: + context: seosnap-cachewarmer + container_name: seosnap_stack_cachewarmer + env_file: ./.env volumes: - - ./seosnap-cachewarmer:/code - ./logs:/code/logs - restart: "no" depends_on: - dashboard - cacheserver @@ -34,42 +24,29 @@ services: - seosnap cacheserver: - build: ./seosnap-cacheserver - image: seosnap-cacheserver - container_name: seosnap_cacheserver - env_file: .env + build: + context: seosnap-cacheserver + container_name: seosnap_stack_cacheserver + env_file: ./.env volumes: - ./cache:/app/cache - ./logs:/app/logs - restart: "unless-stopped" - ports: - - 5000:5000 depends_on: - rendertron networks: - seosnap rendertron: - image: ammobindotca/rendertron - container_name: rendertron - restart: "unless-stopped" - ports: - - 3000:3000 + container_name: snapstack_rendertron + env_file: ./.env networks: - seosnap db: - image: mysql - entrypoint: ['/entrypoint.sh', '--default-authentication-plugin=mysql_native_password'] - environment: - MYSQL_ROOT_HOST: ${DB_ROOT_HOST} - MYSQL_RANDOM_ROOT_PASSWORD: "yes" - MYSQL_DATABASE: ${DB_NAME} - MYSQL_USER: ${DB_USER} - MYSQL_PASSWORD: ${DB_PASS} + container_name: seosnap_stack_db + env_file: ./.env volumes: - ./dev/instance/mariadb:/var/lib/mysql - restart: "unless-stopped" networks: - seosnap diff --git a/seosnap-cacheserver b/seosnap-cacheserver index 7557f0f..7eb442c 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit 7557f0f64ad4e153e574f5f248272b5a18eb80fd +Subproject commit 7eb442cbceb13d6ce88146a29ea2187209569b6e diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index f2d38da..4ef4277 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit f2d38da035ea806182174e48cdce369bc4ab74fc +Subproject commit 4ef4277a9a33d64295532032ca2476a0996b7498 diff --git a/seosnap-dashboard b/seosnap-dashboard index 390120c..0b98c03 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit 390120c9aa15c609f8b02b9712cb1f5787c583b6 +Subproject commit 0b98c03d4531a0b1785ba75998426d920b47c16e From 196429add7cc8daf68a4510ba30f4d1cc71b05c7 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sun, 15 Nov 2020 14:33:07 +0100 Subject: [PATCH 2/5] Updated rendertron to support restricton of image requests --- docker-compose.yml | 4 +++- rendertron-config.json | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 rendertron-config.json diff --git a/docker-compose.yml b/docker-compose.yml index e7f778a..151264f 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,9 +50,11 @@ services: - seosnap rendertron: - image: ammobindotca/rendertron + image: egordm/rendertron:latest container_name: rendertron restart: "unless-stopped" + volumes: + - ./rendertron-config.json:/app/config.json ports: - 3000:3000 networks: diff --git a/rendertron-config.json b/rendertron-config.json new file mode 100644 index 0000000..c6b2956 --- /dev/null +++ b/rendertron-config.json @@ -0,0 +1,3 @@ +{ + "restrictedUrlPattern": ".*(\\.png|\\.jpg|\\.jpeg|\\.gif|\\.webp|\\.mp4)($|\\?)" +} From 16280a823eacc7a01a6e92c961f973bca6faf2a3 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 27 Nov 2020 14:51:47 +0100 Subject: [PATCH 3/5] [FEATURE] Changed default .env dashboard port. Updated dependencies --- .env.example | 2 +- seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index f99d721..13f8ef7 100755 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ RENDERTRON_CACHE_RESOURCE_METHOD=GET RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST= RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST=Set-Cookie,Content-Encoding,Transfer-Encoding -API_URL=http://dashboard:8080/docs/ +API_URL=http://dashboard:80/docs/ API_VER=v1 API_NAME=snaptron API_PASS=Sn@ptron1337 diff --git a/seosnap-cacheserver b/seosnap-cacheserver index 68459df..cc2c259 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit 68459df1d3b7faa49cdebec4e58257967b5072ef +Subproject commit cc2c25906ed2800ec76028accb6b6350cb019a17 diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index baa399c..4ef4277 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit baa399ce624dd203ab6b586112e68305661137b2 +Subproject commit 4ef4277a9a33d64295532032ca2476a0996b7498 diff --git a/seosnap-dashboard b/seosnap-dashboard index d8dca75..2e408ff 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit d8dca75be51f611e1f7129cfc3cea78aedd29b6f +Subproject commit 2e408ffd543d17c8c147b810c0cab70308637587 From 70079629e8a60d80048d825624aa89705b85416b Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sun, 29 Nov 2020 21:19:08 +0100 Subject: [PATCH 4/5] [FEATURE] Added daemon run mode to the makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index e6b67d8..83f457d 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev up: docker-compose ${DC_CONFIGS} -f docker-compose.yml up +daemon: + docker-compose ${DC_CONFIGS} -f docker-compose.yml up -d + build: docker-compose ${DC_CONFIGS} -f docker-compose.yml build From 784592b7adfddb632ca2e2517f715d8a032e5b38 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Sun, 29 Nov 2020 21:37:15 +0100 Subject: [PATCH 5/5] [FEATURE] Prepared submodules for 0.3.0 release --- seosnap-cacheserver | 2 +- seosnap-cachewarmer | 2 +- seosnap-dashboard | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seosnap-cacheserver b/seosnap-cacheserver index cc2c259..6bc6cdf 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit cc2c25906ed2800ec76028accb6b6350cb019a17 +Subproject commit 6bc6cdff99617b3c2374c9936166e2f1e24f9725 diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index 4ef4277..7f931b9 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit 4ef4277a9a33d64295532032ca2476a0996b7498 +Subproject commit 7f931b9d2b962b219a074dbf612fb4ced356f14d diff --git a/seosnap-dashboard b/seosnap-dashboard index 2e408ff..b7830d7 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit 2e408ffd543d17c8c147b810c0cab70308637587 +Subproject commit b7830d7a99fcfc48a072a5ce3acb81065b01f0da