diff --git a/Dockerfile b/Dockerfile index 27ed6c7b..9c7da385 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV CKAN_ENV docker WORKDIR /opt/inventory-app # Install required packages -RUN apt-get -q -y update +RUN apt-get -q -y update --fix-missing RUN apt-get -q -y install \ curl \ build-essential \ @@ -49,7 +49,7 @@ RUN mkdir -p $CKAN_HOME && \ COPY requirements.txt /tmp/ # Install ckan dependencies -RUN $CKAN_HOME/bin/pip install -r /tmp/requirements.txt +RUN $CKAN_HOME/bin/pip --no-cache-dir install -r /tmp/requirements.txt COPY entrypoint-docker.sh / ENTRYPOINT ["/entrypoint-docker.sh"] diff --git a/config/production.ini b/config/production.ini index d23178a9..fb0e762e 100644 --- a/config/production.ini +++ b/config/production.ini @@ -59,7 +59,12 @@ ckan.redis.url = redis://redis:6379/0 ## Site Settings -ckan.site_url = http://localhost:5000 +ckan.site_url = http://localhost:5000/ + +# inside docker we need a custom URL +# https://github.com/ckan/ckan/pull/4879/files#diff-a6a8dc8b17704fdc5f98ec7b3dfe0045R1534 +ckan.datapusher.callback_url_base = http://app:5000/ + #ckan.use_pylons_response_cleanup_middleware = true ## Authorization Settings @@ -102,7 +107,7 @@ solr_url = http://solr:8983/solr/inventory # original plugins # ckan.plugins = usmetadata datajson datastore datapusher stats text_view recline_view googleanalyticsbasic -ckan.plugins = googleanalyticsbasic datastore +ckan.plugins = googleanalyticsbasic datastore datapusher # Define which views should be created by default # (plugins must be loaded in ckan.plugins) @@ -155,7 +160,7 @@ ckan.feeds.author_link = # Make sure you have set up the DataStore ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -ckan.datapusher.url = http://127.0.0.1:8800/ +ckan.datapusher.url = http://datapusher:8800 # Resource Proxy settings # Preview size limit, default: 1MB diff --git a/datapusher/Dockerfile b/datapusher/Dockerfile new file mode 100644 index 00000000..b5bb41ab --- /dev/null +++ b/datapusher/Dockerfile @@ -0,0 +1,39 @@ +FROM keitaro/base:0.4 + +MAINTAINER Keitaro Inc + +ENV APP_DIR=/srv/app +ENV GIT_BRANCH 0.0.15 +ENV GIT_URL https://github.com/ckan/datapusher.git +ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py + +WORKDIR ${APP_DIR} + +RUN apk add --no-cache python \ + py-pip \ + py-gunicorn \ + libffi-dev \ + libressl-dev \ + libxslt && \ + # Temporary packages to build CKAN requirements + apk add --no-cache --virtual .build-deps \ + gcc \ + git \ + util-linux \ + musl-dev \ + python-dev \ + libxml2-dev \ + libxslt-dev + +# Fetch datapusher and install +RUN mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \ + git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \ + cd datapusher && \ + pip install -r requirements.txt && \ + python setup.py develop + +COPY setup ${APP_DIR} + +EXPOSE 8800 +# --reload restats gunicoir when code changes +CMD ["gunicorn", "--reload", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"] \ No newline at end of file diff --git a/datapusher/setup/datapusher_settings.py b/datapusher/setup/datapusher_settings.py new file mode 100644 index 00000000..4002dfae --- /dev/null +++ b/datapusher/setup/datapusher_settings.py @@ -0,0 +1,32 @@ +import uuid + +DEBUG = True +TESTING = False +SECRET_KEY = str(uuid.uuid4()) +USERNAME = str(uuid.uuid4()) +PASSWORD = str(uuid.uuid4()) + +NAME = 'datapusher' + +# database + +SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db' + +# webserver host and port + +HOST = '0.0.0.0' +PORT = 8800 + +# logging + +#FROM_EMAIL = 'server-error@example.com' +#ADMINS = ['yourname@example.com'] # where to send emails + +LOG_FILE = '/tmp/ckan_service.log' +STDERR = True + +# cloud settings +MAX_CONTENT_LENGTH = 73400320 + +# Allow no SSL locally +SSL_VERIFY = False \ No newline at end of file diff --git a/datapusher/setup/wsgi.py b/datapusher/setup/wsgi.py new file mode 100644 index 00000000..b655ee3c --- /dev/null +++ b/datapusher/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +import sys + +import ckanserviceprovider.web as web +web.init() + +from datapusher import jobs + +application = web.app diff --git a/docker-compose.yml b/docker-compose.yml index 5c5e64ab..b709a891 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: command: /opt/inventory-app/start.sh depends_on: - datastore + - datapusher - db - solr environment: @@ -28,6 +29,16 @@ services: volumes: - ./datastore/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + datapusher: + container_name: datapusher-inventory + build: + context: datapusher/ + ports: + - "8800:8800" + # activate if you need to debug datapusher + # volumes: + # - /home/user/my_code/datapusher:/srv/app/src/datapusher + db: image: datagov/catalog-db:inventory2_8 environment: diff --git a/requirements.txt b/requirements.txt index f4f63a1b..6b8971f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ --e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan' +# -e 'git+https://github.com/ckan/ckan.git@ckan-2.8.3#egg=ckan' + +# datapusher hotfix. CKAN has an error +-e git+https://github.com/avdata99/ckan@my_2.8.3#egg=ckan -e git+https://github.com/GSA/ckanext-googleanalyticsbasic#egg=ckanext-googleanalyticsbasic #