CI Tests

.travis.yml

@@ -1,25 +1,47 @@
 dist: xenial # required for Python >= 3.7
 language: generic
+sudo: required
+services:
+  - docker
 
 # Build the docker image
 script:
-  - docker build -t ${TRAVIS_COMMIT} .
+- travis_fold start docker_build
+- docker build -t ${TRAVIS_COMMIT} -t killrvideo-python-local .
+- travis_fold end docker_build
+- travis_fold start docker_dependencies_up
+- docker-compose -f docker-compose.ci.yml up -d
+- sleep 300
+- docker-compose -f docker-compose.ci.yml logs dse-config 
+- travis_fold end docker_dependencies_up
+- travis_fold start test
+- docker run --network killrvideo-python_default killrvideo/killrvideo-integration-tests
+- travis_fold end test
 
 # If successful, see if we need to publish also
 after_success:
-  - test -z $TRAVIS_TAG && travis_terminate 0
-  - docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG}
-  - docker login -u $DOCKER_USER -p $DOCKER_PASS
-  - docker push killrvideo/killrvideo-python:${TRAVIS_TAG}
-  - "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] &&  { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }"
-# Sudo required for doing docker build
-sudo: required
-services:
-  - docker
+- "[ \"$TRAVIS_EVENT_TYPE\" = \"cron\" ] && { echo \"Ignore nightly builds\"; travis_terminate 0; }"
+- test -z $TRAVIS_TAG && { echo "Ignore non-tagged builds"; travis_terminate 0; }
+- docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG}
+- echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+- docker push killrvideo/killrvideo-python:${TRAVIS_TAG}
+- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }"
+- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); }"
+- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); }"
+
+after_failure:
+- travis_fold start logs
+- docker-compose -f docker-compose.ci.yml logs dse-config backend
+- travis_fold end logs
 
 env:
   global:
   # DOCKER_USER
   - secure: "N7FzHIc8S/HyMOmEe9AGlMDDpvPG4+fQMX4J2Ws60PJno/g6h5Gifk0D7DS8zAgOWw7y1U+r8WXefRNGXj1WCFmTrchJhbNoURvqZ/yhrBWoHjEW3f4hxPw0KowAqHf6ZBrhGhoDTPJ+rz4YAnbVjMu10S+1/55EjkrlSyDQ9EwHgTjo63k8hz1dRjMpGzxp4i9XRUc2qQZchFCT+H8c5RQqlQjG4SOhKfp+igVhCWO86U0PoJEMR7y6nYCpSVLdnX2W9oUyQI9VxCh35bnxRVgGCX71eqoyqxS3AQ0LLqY8Yy4PiPzkRCSGFX9qtbgrLd+t5osZsdFyX3YfzmBr3dVCmQwkoHcOM2L3HaKtfPCsni92ukEoVBpU39ajZl8qpCt5QcRn7CpqLQmNeh9xfxlvoyfeR/DTlqKpFy/l6kXgFjrIdssXA5qSiAV873nW4vZerBGkwmWnsvojYA4zHMaT3fvFaFCfT9G2haLGyPF2ZYVpgl40Ficu1w7MrED8Zenou9lOvwDaG+g6vzGTBXwpie7yxPjSrVZi4OK7io574esG3nUwHcqDzEOar1FeBgDbo1I60Dw4C3uwJuM0fbBZo1uZ122DEnwjkzuojJl4r3V+OzFJEF9SgtZTJjZ84yYDXbJ+BjANf72d75c3hwP9Y3ppFunmeKCTcb4aMVs="
   # DOCKER_PASS
   - secure: "F8sNKMia3lYxc7X+vn7448CxX3z0Df06RJPBChM7oAREP2fxvOEzgipgMbHPx4POcdSr60aDSJfDfXpTPRylFkkNK8d733HQdvsLA1lyVoHr6pKRjU8QNz4AMMIxHNhunMTQ5XfJEn9Q+a5qmcsho9HOCA4rfhT/thKIC9QlX/qNme4v27n0vjvqh7cUHU2gV5ykZHf324WBRrzHgYyEe0ieNJ6Tvmxf+Kzm3JB6X20uqst48tS/pPDAkdQPnlu4kWprFwRfKboSjLQjgnnHY+zZLvgvP7kyfO817M5n7iH5WAD1jK4o1stb0tSM/1u4SwssxIyLOKJeclCfY9XkG3DjtNuU+43MUKsI8Q/39fzqW39nxYUCcM4kbW5rWZUCz9PfCs3MOd4xhgX7+mf9xT66HclsJcdbqyrty41rSB0/x8T2LtzoW/Q6UD2GNHimkRu2cTDI2aV1HxbZjqGlFxK5x77CPQAuFxtbnaUH6pSoi3fiYyJiivLLTGzeisA2qpjeSMS2+q3KZIpcqaDIMV4slvfng7gurm2TvtlEtwG1ITJ01dokcXNy+vcw/3go606Aemr/bK2Aot00X0/HDQaX1/v0rJVWXW/c3Gcso6Jzmi1C9vp8gTOr5closfqZFyDJFC1Wuv+vdAzW4MMuwAuc4DnoiwQ6Gm8LamV9oy8="
+
+notifications:
+  slack:
+    rooms:
+      secure: Eh/HI7+7KAUlzFpl9ViMUpExt4YXXWa5rEcwW3h/zIAo/UVghstuoCnk/CWbY/4SyrBeFO7iGcAc396GrBHbkJA25Iz+z3M7VYI/v22di1P8DeWguLNxbe1/wwCAbneYlxLbpPmkOKuFVho5NWDF1QgGszWONRM6DyiFiPlMF84nUcZEHP9exRELGqY64Cg3gfaKMXBn+QSZ5dzNH5cgVrWsGmPSeVBSTWZnZUYHWFKTbgnkY0bs1aHQefRAvl2CCZY/eFPJtcM9oIyIOiuX0NVz7h971tFJdJ9zpTesmaSVOjhCT00mUEJnPPI+zFJxLyIiXHjwROZXzCZL3J21DuWAb3+3ndVe6f47ao/4VNR/Q1TNJGIw5ewkA7RHHW9fegF27OzF+ddfd88ExFvBh1PvJXlAO3c050/2dIqDw7ovhIZWHFchUjSaoV6o5mkgv8/rEu76o3cpdAa78kgZ2tqzDQSquIY55eyHY1vDRusLYii5QZkXzblN9Hm/VvWuM//X99XtoFfnloeLcARWu6ze0fnsBeexK6iQMuUqseLZTjF3tnFqVe2cArA7Swwkh77fRWT3jm35FX1SHaT/kecsMiPY/E7sBfFGHjHVqSfEwMYXhJ6O3aur0UssB+UpuzWu+yBCkf7TienwzimXMrvOGnE++dSOxx6mXRaJBhI=

Dockerfile

@@ -1,35 +1,14 @@
 # Dockerfile
 FROM python:3.7
 
-ARG KILLRVIDEO_DOCKER_IP
-ENV KILLRVIDEO_DOCKER_IP ${KILLRVIDEO_DOCKER_IP}
-
-ARG KILLRVIDEO_HOST_IP
-ENV KILLRVIDEO_HOST_IP ${KILLRVIDEO_HOST_IP}
-
-ARG KILLRVIDEO_DSE_USERNAME
-ENV KILLRVIDEO_DSE_USERNAME ${KILLRVIDEO_DSE_USERNAME}
-
-ARG KILLRVIDEO_DSE_PASSWORD
-ENV KILLRVIDEO_DSE_PASSWORD ${KILLRVIDEO_DSE_PASSWORD}
+COPY killrvideo/ requirements.txt /app/
+WORKDIR /app
 
 # Install app dependencies
-RUN pip install dse-driver
-RUN pip install dse-graph  
-RUN pip install protobuf
-RUN pip install grpcio
-RUN pip install python-etcd
-RUN pip install time-uuid
-RUN pip install validate-email
-RUN pip install sortedcontainers
-RUN pip install nltk
-RUN python -m nltk.downloader stopwords
-RUN pip install kafka-python
+RUN pip install -r requirements.txt \
+    && python -m nltk.downloader stopwords
 
 # Create app directory
-COPY killrvideo/ /app
-WORKDIR /app
-
 ENV PYTHONPATH "${PYTHONPATH}:/${WORKDIR}"
 
 EXPOSE 50101

docker-compose.ci.yml

@@ -0,0 +1,39 @@
+version: '3'
+services:
+  backend:
+    image: killrvideo-python-local
+    depends_on:
+      - dse
+
+  dse: 
+    image: datastax/dse-server:6.7.4
+    command: [ -s -g ]
+    environment:
+      DS_LICENSE: accept
+    cap_add:
+      - IPC_LOCK
+    ulimits:
+      memlock: -1
+
+  dse-config: 
+    image: killrvideo/killrvideo-dse-config:3
+    depends_on:
+      - dse
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - 2181:2181
+
+  # Start a Kafka Broker for Killrvideo
+  kafka:
+    image: wurstmeister/kafka
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "topic-kv-videoCreation:1:1,topic-kv-userCreation:1:1,topic-kv-videoRating:1:1,topic-kv-commentCreation:1:1"
+    ports:
+      - 9092:9092

docker-compose.yaml

@@ -1,30 +1,10 @@
 version: '3'
 
-#
-# docker-compose.yaml
-#   Simple compose file for running the Python services in Docker along with other containers comprising the
-#   KillrVideo.
-#   For more complex configuration options including metrics, external volumes, OpsCenter,
-#   etc., see https://github.com/KillrVideo/killrvideo-docker-common
-#
-#   Note: the default configuration here uses the LATEST version of the killrvideo-python
-#   Docker image, which can be built using scripts/docker_build.sh.
-#
-
 services:
 
-  web: # Web Interface
-    image: killrvideo/killrvideo-web:3.0.1
-    ports:
-      - "3000:3000" # Exposes port to be available externally
-    depends_on:
-      - backend # Needs Backend to be running
-    environment:
-      KILLRVIDEO_LOGGING_LEVEL: debug # More logging
-
   # Service implementation - use docker-compose-backend-external.yaml if running services in IDE
   backend: 
-    image: killrvideo/killrvideo-python
+    build: .
     ports:
       - "50101:50101" # Exposes port to be available.
     depends_on:
@@ -33,6 +13,15 @@ services:
     environment:
       KILLRVIDEO_LOGGING_LEVEL: debug # More logging
 
+  web: # Web Interface
+    image: killrvideo/killrvideo-web:3
+    ports:
+      - "3000:3000" # Exposes port to be available externally
+    depends_on:
+      - backend # Needs Backend to be running
+    environment:
+      KILLRVIDEO_LOGGING_LEVEL: debug # More logging
+
   # Datastax Studio provides convenient web interface to work directly with DataStax Enterprise
   studio:  
     image: datastax/dse-studio:6.7.0
@@ -60,14 +49,14 @@ services:
 
   # One-Time Bootstrap Container, configures DSE to have required keyspaces etc.
   dse-config: 
-    image: killrvideo/killrvideo-dse-config:3.0.0
+    image: killrvideo/killrvideo-dse-config:3
     depends_on:
       - dse # Needs DSE to be running
 
   # Sample Data Generator, imitates behaviour of users on the killrVideo website. 
   # Adds comments and rates videos, upload new videos and so on.
   generator: 
-    image: killrvideo/killrvideo-generator:3.0.1
+    image: killrvideo/killrvideo-generator:3
     depends_on:
       - backend # Needs Backend to be running
     environment: