Upgrade datapusher

Dockerfile

@@ -9,7 +9,7 @@ ENV CKAN_ENV docker
 WORKDIR /opt/inventory-app
 
 # Install required packages
-RUN apt-get -q -y update
+RUN apt-get -q -y update --fix-missing
 RUN apt-get -q -y install \
   curl \
   build-essential \
@@ -49,7 +49,7 @@ RUN mkdir -p $CKAN_HOME && \
 COPY requirements.txt /tmp/
 
 # Install ckan dependencies
-RUN $CKAN_HOME/bin/pip install -r /tmp/requirements.txt
+RUN $CKAN_HOME/bin/pip --no-cache-dir install -r /tmp/requirements.txt
 
 COPY entrypoint-docker.sh /
 ENTRYPOINT ["/entrypoint-docker.sh"]

config/production.ini

@@ -59,7 +59,12 @@ ckan.redis.url = redis://redis:6379/0
 
 ## Site Settings
 
-ckan.site_url = http://localhost:5000
+ckan.site_url = http://localhost:5000/
+
+# inside docker we need a custom URL
+# https://github.com/ckan/ckan/pull/4879/files#diff-a6a8dc8b17704fdc5f98ec7b3dfe0045R1534 
+ckan.datapusher.callback_url_base = http://app:5000/
+
 #ckan.use_pylons_response_cleanup_middleware = true
 
 ## Authorization Settings
@@ -102,7 +107,7 @@ solr_url = http://solr:8983/solr/inventory
 # original plugins
 # ckan.plugins = usmetadata datajson datastore datapusher stats text_view recline_view googleanalyticsbasic
 
-ckan.plugins = googleanalyticsbasic datastore
+ckan.plugins = googleanalyticsbasic datastore datapusher
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
@@ -155,7 +160,7 @@ ckan.feeds.author_link =
 # Make sure you have set up the DataStore
 
 ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-ckan.datapusher.url = http://127.0.0.1:8800/
+ckan.datapusher.url = http://datapusher:8800
 
 # Resource Proxy settings
 # Preview size limit, default: 1MB

datapusher/Dockerfile

@@ -0,0 +1,39 @@
+FROM keitaro/base:0.4
+
+MAINTAINER Keitaro Inc <info@keitaro.info>
+
+ENV APP_DIR=/srv/app
+ENV GIT_BRANCH 0.0.15
+ENV GIT_URL https://github.com/ckan/datapusher.git
+ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
+
+WORKDIR ${APP_DIR}
+
+RUN apk add --no-cache python \
+    py-pip \
+    py-gunicorn \
+    libffi-dev \
+    libressl-dev \
+    libxslt && \
+    # Temporary packages to build CKAN requirements
+    apk add --no-cache --virtual .build-deps \
+    gcc \
+    git \
+    util-linux \
+    musl-dev \
+    python-dev \
+    libxml2-dev \
+    libxslt-dev 
+
+# Fetch datapusher and install
+RUN mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \
+    git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \
+    cd datapusher && \
+    pip install -r requirements.txt && \
+    python setup.py develop
+
+COPY setup ${APP_DIR}
+
+EXPOSE 8800
+# --reload restats gunicoir when code changes
+CMD ["gunicorn", "--reload", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"]

datapusher/setup/datapusher_settings.py

@@ -0,0 +1,32 @@
+import uuid
+
+DEBUG = True
+TESTING = False
+SECRET_KEY = str(uuid.uuid4())
+USERNAME = str(uuid.uuid4())
+PASSWORD = str(uuid.uuid4())
+
+NAME = 'datapusher'
+
+# database
+
+SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db'
+
+# webserver host and port
+
+HOST = '0.0.0.0'
+PORT = 8800
+
+# logging
+
+#FROM_EMAIL = 'server-error@example.com'
+#ADMINS = ['yourname@example.com']  # where to send emails
+
+LOG_FILE = '/tmp/ckan_service.log'
+STDERR = True
+
+# cloud settings
+MAX_CONTENT_LENGTH = 73400320
+
+# Allow no SSL locally
+SSL_VERIFY = False

datapusher/setup/wsgi.py

@@ -0,0 +1,9 @@
+import os
+import sys
+
+import ckanserviceprovider.web as web
+web.init()
+
+from datapusher import jobs
+
+application = web.app

docker-compose.yml

@@ -5,6 +5,7 @@ services:
     command: /opt/inventory-app/start.sh
     depends_on:
       - datastore
+      - datapusher
       - db
       - solr
     environment:
@@ -28,6 +29,16 @@ services:
     volumes:
       - ./datastore/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
+  datapusher:
+    container_name: datapusher-inventory
+    build:
+      context: datapusher/
+    ports:
+      - "8800:8800"
+    # activate if you need to debug datapusher
+    # volumes:
+    #   - /home/user/my_code/datapusher:/srv/app/src/datapusher
+
   db:
     image: datagov/catalog-db:inventory2_8
     environment:

requirements.txt

@@ -1,4 +1,7 @@
--e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan'
+# -e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan'
+
+# datapusher hotfix. CKAN has an error
+-e git+https://github.com/avdata99/ckan@my_2.8.3#egg=ckan
 -e  git+https://github.com/GSA/ckanext-googleanalyticsbasic#egg=ckanext-googleanalyticsbasic
 
 #