From edfd4c28f5d4dcce2c0cd5c280b92cff8d2c5bf5 Mon Sep 17 00:00:00 2001 From: Andres Vazquez Date: Thu, 12 Mar 2020 12:01:13 -0300 Subject: [PATCH 1/2] Start testing datapusher --- Dockerfile | 2 +- config/production.ini | 4 +-- datapusher/Dockerfile | 39 +++++++++++++++++++++++++ datapusher/setup/datapusher_settings.py | 32 ++++++++++++++++++++ datapusher/setup/wsgi.py | 9 ++++++ docker-compose.yml | 8 +++++ 6 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 datapusher/Dockerfile create mode 100644 datapusher/setup/datapusher_settings.py create mode 100644 datapusher/setup/wsgi.py diff --git a/Dockerfile b/Dockerfile index 27ed6c7b..9550f8c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV CKAN_ENV docker WORKDIR /opt/inventory-app # Install required packages -RUN apt-get -q -y update +RUN apt-get -q -y update --fix-missing RUN apt-get -q -y install \ curl \ build-essential \ diff --git a/config/production.ini b/config/production.ini index d23178a9..af8d2a43 100644 --- a/config/production.ini +++ b/config/production.ini @@ -102,7 +102,7 @@ solr_url = http://solr:8983/solr/inventory # original plugins # ckan.plugins = usmetadata datajson datastore datapusher stats text_view recline_view googleanalyticsbasic -ckan.plugins = googleanalyticsbasic datastore +ckan.plugins = googleanalyticsbasic datastore datapusher # Define which views should be created by default # (plugins must be loaded in ckan.plugins) @@ -155,7 +155,7 @@ ckan.feeds.author_link = # Make sure you have set up the DataStore ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -ckan.datapusher.url = http://127.0.0.1:8800/ +ckan.datapusher.url = http://datapusher:8800 # Resource Proxy settings # Preview size limit, default: 1MB diff --git a/datapusher/Dockerfile b/datapusher/Dockerfile new file mode 100644 index 00000000..eee33439 --- /dev/null +++ b/datapusher/Dockerfile @@ -0,0 +1,39 @@ +FROM keitaro/base:0.4 + +MAINTAINER Keitaro Inc + +ENV APP_DIR=/srv/app +ENV GIT_BRANCH 0.0.15 +ENV GIT_URL https://github.com/ckan/datapusher.git +ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py + +WORKDIR ${APP_DIR} + +RUN apk add --no-cache python \ + py-pip \ + py-gunicorn \ + libffi-dev \ + libressl-dev \ + libxslt && \ + # Temporary packages to build CKAN requirements + apk add --no-cache --virtual .build-deps \ + gcc \ + git \ + musl-dev \ + python-dev \ + libxml2-dev \ + libxslt-dev && \ + # Fetch datapusher and install + mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \ + git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \ + cd datapusher && \ + python setup.py install && \ + pip install --no-cache-dir -r requirements.txt && \ + # Remove temporary packages and files + apk del .build-deps && \ + rm -rf ${APP_DIR}/src + +COPY setup ${APP_DIR} + +EXPOSE 8800 +CMD ["gunicorn", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"] diff --git a/datapusher/setup/datapusher_settings.py b/datapusher/setup/datapusher_settings.py new file mode 100644 index 00000000..511b6429 --- /dev/null +++ b/datapusher/setup/datapusher_settings.py @@ -0,0 +1,32 @@ +import uuid + +DEBUG = False +TESTING = False +SECRET_KEY = str(uuid.uuid4()) +USERNAME = str(uuid.uuid4()) +PASSWORD = str(uuid.uuid4()) + +NAME = 'datapusher' + +# database + +SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db' + +# webserver host and port + +HOST = '0.0.0.0' +PORT = 8800 + +# logging + +#FROM_EMAIL = 'server-error@example.com' +#ADMINS = ['yourname@example.com'] # where to send emails + +#LOG_FILE = '/tmp/ckan_service.log' +STDERR = True + +# cloud settings +MAX_CONTENT_LENGTH = 73400320 + +# Allow no SSL locally +SSL_VERIFY = False \ No newline at end of file diff --git a/datapusher/setup/wsgi.py b/datapusher/setup/wsgi.py new file mode 100644 index 00000000..8701ffa5 --- /dev/null +++ b/datapusher/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +import sys + +import ckanserviceprovider.web as web +web.init() + +import datapusher.jobs as jobs + +application = web.app diff --git a/docker-compose.yml b/docker-compose.yml index 5c5e64ab..30508e72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: command: /opt/inventory-app/start.sh depends_on: - datastore + - datapusher - db - solr environment: @@ -28,6 +29,13 @@ services: volumes: - ./datastore/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + datapusher: + container_name: datapusher-inventory + build: + context: datapusher/ + ports: + - "8800:8800" + db: image: datagov/catalog-db:inventory2_8 environment: From 1498ee88ce4881e3f6606ec884c137c5394897ef Mon Sep 17 00:00:00 2001 From: Andres Vazquez Date: Fri, 13 Mar 2020 21:18:54 -0300 Subject: [PATCH 2/2] Environment ready for debug and run datapusher --- Dockerfile | 2 +- config/production.ini | 7 ++++++- datapusher/Dockerfile | 18 +++++++++--------- datapusher/setup/datapusher_settings.py | 4 ++-- datapusher/setup/wsgi.py | 2 +- docker-compose.yml | 3 +++ requirements.txt | 5 ++++- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9550f8c2..9c7da385 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ RUN mkdir -p $CKAN_HOME && \ COPY requirements.txt /tmp/ # Install ckan dependencies -RUN $CKAN_HOME/bin/pip install -r /tmp/requirements.txt +RUN $CKAN_HOME/bin/pip --no-cache-dir install -r /tmp/requirements.txt COPY entrypoint-docker.sh / ENTRYPOINT ["/entrypoint-docker.sh"] diff --git a/config/production.ini b/config/production.ini index af8d2a43..fb0e762e 100644 --- a/config/production.ini +++ b/config/production.ini @@ -59,7 +59,12 @@ ckan.redis.url = redis://redis:6379/0 ## Site Settings -ckan.site_url = http://localhost:5000 +ckan.site_url = http://localhost:5000/ + +# inside docker we need a custom URL +# https://github.com/ckan/ckan/pull/4879/files#diff-a6a8dc8b17704fdc5f98ec7b3dfe0045R1534 +ckan.datapusher.callback_url_base = http://app:5000/ + #ckan.use_pylons_response_cleanup_middleware = true ## Authorization Settings diff --git a/datapusher/Dockerfile b/datapusher/Dockerfile index eee33439..b5bb41ab 100644 --- a/datapusher/Dockerfile +++ b/datapusher/Dockerfile @@ -19,21 +19,21 @@ RUN apk add --no-cache python \ apk add --no-cache --virtual .build-deps \ gcc \ git \ + util-linux \ musl-dev \ python-dev \ libxml2-dev \ - libxslt-dev && \ - # Fetch datapusher and install - mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \ + libxslt-dev + +# Fetch datapusher and install +RUN mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \ git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \ cd datapusher && \ - python setup.py install && \ - pip install --no-cache-dir -r requirements.txt && \ - # Remove temporary packages and files - apk del .build-deps && \ - rm -rf ${APP_DIR}/src + pip install -r requirements.txt && \ + python setup.py develop COPY setup ${APP_DIR} EXPOSE 8800 -CMD ["gunicorn", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"] +# --reload restats gunicoir when code changes +CMD ["gunicorn", "--reload", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"] \ No newline at end of file diff --git a/datapusher/setup/datapusher_settings.py b/datapusher/setup/datapusher_settings.py index 511b6429..4002dfae 100644 --- a/datapusher/setup/datapusher_settings.py +++ b/datapusher/setup/datapusher_settings.py @@ -1,6 +1,6 @@ import uuid -DEBUG = False +DEBUG = True TESTING = False SECRET_KEY = str(uuid.uuid4()) USERNAME = str(uuid.uuid4()) @@ -22,7 +22,7 @@ #FROM_EMAIL = 'server-error@example.com' #ADMINS = ['yourname@example.com'] # where to send emails -#LOG_FILE = '/tmp/ckan_service.log' +LOG_FILE = '/tmp/ckan_service.log' STDERR = True # cloud settings diff --git a/datapusher/setup/wsgi.py b/datapusher/setup/wsgi.py index 8701ffa5..b655ee3c 100644 --- a/datapusher/setup/wsgi.py +++ b/datapusher/setup/wsgi.py @@ -4,6 +4,6 @@ import ckanserviceprovider.web as web web.init() -import datapusher.jobs as jobs +from datapusher import jobs application = web.app diff --git a/docker-compose.yml b/docker-compose.yml index 30508e72..b709a891 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,9 @@ services: context: datapusher/ ports: - "8800:8800" + # activate if you need to debug datapusher + # volumes: + # - /home/user/my_code/datapusher:/srv/app/src/datapusher db: image: datagov/catalog-db:inventory2_8 diff --git a/requirements.txt b/requirements.txt index f4f63a1b..6b8971f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ --e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan' +# -e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan' + +# datapusher hotfix. CKAN has an error +-e git+https://github.com/avdata99/ckan@my_2.8.3#egg=ckan -e git+https://github.com/GSA/ckanext-googleanalyticsbasic#egg=ckanext-googleanalyticsbasic #