Merge pull request #18 from experius/release/0.4.0

Release/0.4.0

.env.example

@@ -1,3 +1,5 @@
+TAG=latest
+
 RENDERTRON_CACHE_DEBUG=0
 RENDERTRON_CACHE_LOCK_TIMEOUT=1
 RENDERTRON_CACHE_ROOT=./cache
@@ -6,6 +8,7 @@ RENDERTRON_CACHE_RESOURCE_URL=http://rendertron:3000/render
 RENDERTRON_CACHE_RESOURCE_METHOD=GET
 RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST=
 RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST=Set-Cookie,Content-Encoding,Transfer-Encoding
+RENDERTRON_MOBILE_REGEX=.*Mobile.*
 
 API_URL=http://dashboard:80/docs/
 API_VER=v1
@@ -33,3 +36,10 @@ ADMIN_PASS=Sn@ptron1337
 ADMIN_SECRET="https://miniwebtool.com/django-secret-key-generator/"
 ADMIN_DEBUG=1
 ADMIN_LOG_LEVEL=ERROR
+
+EMAIL_HOST=localhost
+EMAIL_HOST_USER=
+EMAIL_HOST_PASSWORD=
+EMAIL_PORT=25
+EMAIL_USE_TLS=0
+EMAIL_USE_SSL=0

.github/workflows/release.yml

@@ -3,6 +3,7 @@ name: Release
 on:
   push:
     branches:
+      - master
       - develop
 
 jobs:

.gitignore

@@ -140,4 +140,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
+/release/.env
+/release/.env.example
+/release/*.zip

Makefile

@@ -1,11 +1,16 @@
 #!make
+.PHONY: all release
+
 DC_CONFIGS = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.yml -f seosnap-cachewarmer/docker-compose.yml -f seosnap-dashboard/docker-compose.yml
 DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev.yml -f seosnap-cachewarmer/docker-compose.dev.yml -f seosnap-dashboard/docker-compose.dev.yml
 
 
 up: 
 	docker-compose ${DC_CONFIGS} -f docker-compose.yml up
 
+down:
+	docker-compose ${DC_CONFIGS} -f docker-compose.yml down
+
 daemon:
 	docker-compose ${DC_CONFIGS} -f docker-compose.yml up -d
 
@@ -34,3 +39,10 @@ develop:
 	git submodule foreach --recursive git fetch origin develop
 	git submodule foreach --recursive git checkout develop
 	echo "Everything is now up to date"
+
+
+release:
+	python dev/scripts/release_config.py --configs docker-compose.yml seosnap-cacheserver/docker-compose.yml seosnap-cachewarmer/docker-compose.yml seosnap-dashboard/docker-compose.yml docker-compose.yml
+	rm -rf release/cache release/logs
+	rm -f release/release.zip
+	cd release && zip release.zip * .env.example

README.md

@@ -4,6 +4,14 @@ Setup for the whole seosnap stack including dashboard, cache server and cache wa
  page caching PWA's.
 
 # Installation
+## For usage
+* Download and Extract release.zip from
+  * [Releases](https://github.com/experius/SeoSnap/releases)
+  * [Development Build](https://github.com/experius/SeoSnap/releases/tag/latest)
+* Run `./install.sh` or create a .env file manually
+* Start the Seosnap stack with `docker-compose up`
+
+## For development
 ```
 # Clone
 git clone --recursive git@github.com:experius/SeoSnap.git
@@ -16,7 +24,6 @@ make up
 # Usage
 * Dashboard: http://127.0.0.1:8080/ (default login: snaptron/Sn@ptron1337)
 * API Docs: http://127.0.0.1:8080/docs
-* PHPMyAdmin: http://127.0.0.1:8081/
 * Cache Server: http://127.0.0.1:5000/render/\<your url\>
 
 Logs directory ./logs
@@ -39,50 +46,14 @@ Check the nginx.conf in the example folder
 
 ![diagram](https://github.com/experius/SeoSnap/raw/master/assets/diagram.png)
 
-### Dashboard
+### [Dashboard](https://github.com/experius/SeoSnap-Dashboard)
 In the dashboard you add the website url along with the website sitemap that you want to make 'SeoSnaps' off.
 
-### Crawler
+### [Crawler / Cache Warmer](https://github.com/experius/SeoSnap-Cache-Warmer)
 When the crawler is started it connects with the dashboard api. It uses scrapy to crawl the sitemap. The scrapy results are send to the administration/dashboard. Scrapy requests are send to the cache server. In a similar way that you would do a request to rendertron. 
 
-### Cache Server
+### [Cache Server](https://github.com/experius/SeoSnap-Cache-Server)
 The cache server is a simple file caching server. If a file exist with the content of the page it serves the html from the file. If not, it renders the requested url with rendertron and saves the html output in a file. To refresh the cache the cache-warmer uses PUT requests instead of GET. This will force update from the cache file.
 
-# Build with
-![diagram](https://github.com/experius/SeoSnap/raw/master/assets/software.png)
-
-## Usage cache warmer [See](https://github.com/experius/SeoSnap-Cache-Warmer/blob/master/README.md)
-### Commands
-#### Cache
-Handles caching of pages associated to given website
-```
-Usage: crawl.py cache [OPTIONS] WEBSITE_IDS
-
-Options:
-  --follow_next BOOLEAN  Follows rel-next links if enabled
-  --recache BOOLEAN      Recached all pages instead of not yet cached ones
-  --use_queue BOOLEAN    Cache urls from the queue instead of the sitemap
-  --load BOOLEAN         Whether already loaded urls should be scraped instead
-  --help                 Show this message and exit.
-```
-
-#### Clean
-Handles cleaning of the dashboard queue
-```
-Usage: crawl.py clean [OPTIONS] WEBSITE_IDS
-
-Options:
-  --help  Show this message and exit.
-```
-
-### Examples
-```
-# Cache the sitemap of website 1
-make warm A="cache 1"
-
-# Cache requests in queue for websites 1 and 2
-make warm A="cache 1,2 use_queue=true"
-
-# Clean the queue for websites 1 and 2
-make warm A="clean 1,2"
-```
+# Built with
+![diagram](https://github.com/experius/SeoSnap/raw/master/assets/software.png)

dev/scripts/release_config.py

@@ -0,0 +1,81 @@
+import yaml
+from compose import config
+from compose.config.config import ConfigDetails, ConfigFile
+from compose.config.serialize import serialize_config
+
+import os
+import argparse
+from shutil import copyfile
+
+parser = argparse.ArgumentParser(description='Seosnap Release Script')
+parser.add_argument('--configs', type=str, nargs='+',  help='Docker compose config files')
+parser.add_argument('--tag', default='latest', help='Tag for the release containers')
+args = parser.parse_args()
+
+CONFIGS = args.configs
+TAG = args.tag
+OUTPUT_DIR = 'release'
+WORKDIR = os.path.abspath(os.curdir)
+
+REPLACE_MOUNTS = {
+    './seosnap-cacheserver/rendertron-config.json': './rendertron-config.json'
+}
+
+if not os.path.exists(os.path.join(WORKDIR, '.env')):
+    copyfile(
+        os.path.join(WORKDIR, './.env.example'),
+        os.path.join(WORKDIR, './.env'),
+    )
+
+configs = []
+for file in CONFIGS:
+    print(f'Reading file: {file}')
+    with open(file, 'r') as f:
+        configs.append(ConfigFile(None, yaml.safe_load(f.read())))
+
+print('Building config')
+env = config.environment.Environment()
+details = ConfigDetails(
+    WORKDIR,
+    configs,
+    env
+)
+cfg = config.load(
+    details,
+    False
+)
+
+
+def relativize(path: str) -> str:
+    result = f'./{os.path.relpath(path, WORKDIR)}' if path and path.startswith(WORKDIR) else path
+    if result in REPLACE_MOUNTS:
+        result = REPLACE_MOUNTS[result]
+    return result
+
+
+print('Preprocessing config')
+for service in cfg.services:
+    print(f'\tPreprocessing service: {service["name"]}')
+    if 'build' in service:
+        service.pop('build')
+
+    for i, volume in enumerate(service['volumes']):
+        service['volumes'][i] = volume._replace(
+            internal=relativize(volume.internal),
+            external=relativize(volume.external)
+        )
+
+
+print(f'Writing composer file')
+with open(os.path.join(OUTPUT_DIR, 'docker-compose.yml'), 'w') as f:
+    f.write(serialize_config(cfg, None, False))
+
+copyfile(
+    os.path.join(WORKDIR, './seosnap-cacheserver/rendertron-config.json'),
+    os.path.join(OUTPUT_DIR, './rendertron-config.json'),
+)
+
+copyfile(
+    os.path.join(WORKDIR, './.env.example'),
+    os.path.join(OUTPUT_DIR, './.env.example'),
+)

dev/scripts/requirements.txt

@@ -0,0 +1 @@
+docker-compose

docker-compose.yml

@@ -47,8 +47,6 @@ services:
   db:
     container_name: seosnap_stack_db
     env_file: ./.env
-    volumes:
-      - ./dev/instance/mariadb:/var/lib/mysql
     networks:
       - seosnap