diff --git a/.travis.yml b/.travis.yml index ff38c17..83fdbd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,38 @@ dist: xenial # required for Python >= 3.7 language: generic +sudo: required +services: + - docker # Build the docker image script: - - docker build -t ${TRAVIS_COMMIT} . +- travis_fold start docker_build +- docker build -t ${TRAVIS_COMMIT} -t killrvideo-python-local . +- travis_fold end docker_build +- travis_fold start docker_dependencies_up +- docker-compose -f docker-compose.ci.yml up -d +- sleep 300 +- docker-compose -f docker-compose.ci.yml logs dse-config +- travis_fold end docker_dependencies_up +- travis_fold start test +- docker run --network killrvideo-python_default killrvideo/killrvideo-integration-tests +- travis_fold end test # If successful, see if we need to publish also after_success: - - test -z $TRAVIS_TAG && travis_terminate 0 - - docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG} - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - docker push killrvideo/killrvideo-python:${TRAVIS_TAG} - - "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" -# Sudo required for doing docker build -sudo: required -services: - - docker +- "[ \"$TRAVIS_EVENT_TYPE\" = \"cron\" ] && { echo \"Ignore nightly builds\"; travis_terminate 0; }" +- test -z $TRAVIS_TAG && { echo "Ignore non-tagged builds"; travis_terminate 0; } +- docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG} +- echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin +- docker push killrvideo/killrvideo-python:${TRAVIS_TAG} +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); }" +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); }" + +after_failure: +- travis_fold start logs +- docker-compose -f docker-compose.ci.yml logs dse-config backend +- travis_fold end logs env: global: @@ -23,3 +40,8 @@ env: - secure: "N7FzHIc8S/HyMOmEe9AGlMDDpvPG4+fQMX4J2Ws60PJno/g6h5Gifk0D7DS8zAgOWw7y1U+r8WXefRNGXj1WCFmTrchJhbNoURvqZ/yhrBWoHjEW3f4hxPw0KowAqHf6ZBrhGhoDTPJ+rz4YAnbVjMu10S+1/55EjkrlSyDQ9EwHgTjo63k8hz1dRjMpGzxp4i9XRUc2qQZchFCT+H8c5RQqlQjG4SOhKfp+igVhCWO86U0PoJEMR7y6nYCpSVLdnX2W9oUyQI9VxCh35bnxRVgGCX71eqoyqxS3AQ0LLqY8Yy4PiPzkRCSGFX9qtbgrLd+t5osZsdFyX3YfzmBr3dVCmQwkoHcOM2L3HaKtfPCsni92ukEoVBpU39ajZl8qpCt5QcRn7CpqLQmNeh9xfxlvoyfeR/DTlqKpFy/l6kXgFjrIdssXA5qSiAV873nW4vZerBGkwmWnsvojYA4zHMaT3fvFaFCfT9G2haLGyPF2ZYVpgl40Ficu1w7MrED8Zenou9lOvwDaG+g6vzGTBXwpie7yxPjSrVZi4OK7io574esG3nUwHcqDzEOar1FeBgDbo1I60Dw4C3uwJuM0fbBZo1uZ122DEnwjkzuojJl4r3V+OzFJEF9SgtZTJjZ84yYDXbJ+BjANf72d75c3hwP9Y3ppFunmeKCTcb4aMVs=" # DOCKER_PASS - secure: "F8sNKMia3lYxc7X+vn7448CxX3z0Df06RJPBChM7oAREP2fxvOEzgipgMbHPx4POcdSr60aDSJfDfXpTPRylFkkNK8d733HQdvsLA1lyVoHr6pKRjU8QNz4AMMIxHNhunMTQ5XfJEn9Q+a5qmcsho9HOCA4rfhT/thKIC9QlX/qNme4v27n0vjvqh7cUHU2gV5ykZHf324WBRrzHgYyEe0ieNJ6Tvmxf+Kzm3JB6X20uqst48tS/pPDAkdQPnlu4kWprFwRfKboSjLQjgnnHY+zZLvgvP7kyfO817M5n7iH5WAD1jK4o1stb0tSM/1u4SwssxIyLOKJeclCfY9XkG3DjtNuU+43MUKsI8Q/39fzqW39nxYUCcM4kbW5rWZUCz9PfCs3MOd4xhgX7+mf9xT66HclsJcdbqyrty41rSB0/x8T2LtzoW/Q6UD2GNHimkRu2cTDI2aV1HxbZjqGlFxK5x77CPQAuFxtbnaUH6pSoi3fiYyJiivLLTGzeisA2qpjeSMS2+q3KZIpcqaDIMV4slvfng7gurm2TvtlEtwG1ITJ01dokcXNy+vcw/3go606Aemr/bK2Aot00X0/HDQaX1/v0rJVWXW/c3Gcso6Jzmi1C9vp8gTOr5closfqZFyDJFC1Wuv+vdAzW4MMuwAuc4DnoiwQ6Gm8LamV9oy8=" + +notifications: + slack: + rooms: + secure: Eh/HI7+7KAUlzFpl9ViMUpExt4YXXWa5rEcwW3h/zIAo/UVghstuoCnk/CWbY/4SyrBeFO7iGcAc396GrBHbkJA25Iz+z3M7VYI/v22di1P8DeWguLNxbe1/wwCAbneYlxLbpPmkOKuFVho5NWDF1QgGszWONRM6DyiFiPlMF84nUcZEHP9exRELGqY64Cg3gfaKMXBn+QSZ5dzNH5cgVrWsGmPSeVBSTWZnZUYHWFKTbgnkY0bs1aHQefRAvl2CCZY/eFPJtcM9oIyIOiuX0NVz7h971tFJdJ9zpTesmaSVOjhCT00mUEJnPPI+zFJxLyIiXHjwROZXzCZL3J21DuWAb3+3ndVe6f47ao/4VNR/Q1TNJGIw5ewkA7RHHW9fegF27OzF+ddfd88ExFvBh1PvJXlAO3c050/2dIqDw7ovhIZWHFchUjSaoV6o5mkgv8/rEu76o3cpdAa78kgZ2tqzDQSquIY55eyHY1vDRusLYii5QZkXzblN9Hm/VvWuM//X99XtoFfnloeLcARWu6ze0fnsBeexK6iQMuUqseLZTjF3tnFqVe2cArA7Swwkh77fRWT3jm35FX1SHaT/kecsMiPY/E7sBfFGHjHVqSfEwMYXhJ6O3aur0UssB+UpuzWu+yBCkf7TienwzimXMrvOGnE++dSOxx6mXRaJBhI= diff --git a/Dockerfile b/Dockerfile index 0147e46..2aca02d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,14 @@ # Dockerfile FROM python:3.7 -ARG KILLRVIDEO_DOCKER_IP -ENV KILLRVIDEO_DOCKER_IP ${KILLRVIDEO_DOCKER_IP} - -ARG KILLRVIDEO_HOST_IP -ENV KILLRVIDEO_HOST_IP ${KILLRVIDEO_HOST_IP} - -ARG KILLRVIDEO_DSE_USERNAME -ENV KILLRVIDEO_DSE_USERNAME ${KILLRVIDEO_DSE_USERNAME} - -ARG KILLRVIDEO_DSE_PASSWORD -ENV KILLRVIDEO_DSE_PASSWORD ${KILLRVIDEO_DSE_PASSWORD} +COPY killrvideo/ requirements.txt /app/ +WORKDIR /app # Install app dependencies -RUN pip install dse-driver -RUN pip install dse-graph -RUN pip install protobuf -RUN pip install grpcio -RUN pip install python-etcd -RUN pip install time-uuid -RUN pip install validate-email -RUN pip install sortedcontainers -RUN pip install nltk -RUN python -m nltk.downloader stopwords -RUN pip install kafka-python +RUN pip install -r requirements.txt \ + && python -m nltk.downloader stopwords # Create app directory -COPY killrvideo/ /app -WORKDIR /app - ENV PYTHONPATH "${PYTHONPATH}:/${WORKDIR}" EXPOSE 50101 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..b75150d --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,39 @@ +version: '3' +services: + backend: + image: killrvideo-python-local + depends_on: + - dse + + dse: + image: datastax/dse-server:6.7.4 + command: [ -s -g ] + environment: + DS_LICENSE: accept + cap_add: + - IPC_LOCK + ulimits: + memlock: -1 + + dse-config: + image: killrvideo/killrvideo-dse-config:3 + depends_on: + - dse + + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 + + # Start a Kafka Broker for Killrvideo + kafka: + image: wurstmeister/kafka + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "topic-kv-videoCreation:1:1,topic-kv-userCreation:1:1,topic-kv-videoRating:1:1,topic-kv-commentCreation:1:1" + ports: + - 9092:9092 diff --git a/docker-compose.yaml b/docker-compose.yaml index d3f1a1f..27dcba7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,30 +1,10 @@ version: '3' -# -# docker-compose.yaml -# Simple compose file for running the Python services in Docker along with other containers comprising the -# KillrVideo. -# For more complex configuration options including metrics, external volumes, OpsCenter, -# etc., see https://github.com/KillrVideo/killrvideo-docker-common -# -# Note: the default configuration here uses the LATEST version of the killrvideo-python -# Docker image, which can be built using scripts/docker_build.sh. -# - services: - web: # Web Interface - image: killrvideo/killrvideo-web:3.0.1 - ports: - - "3000:3000" # Exposes port to be available externally - depends_on: - - backend # Needs Backend to be running - environment: - KILLRVIDEO_LOGGING_LEVEL: debug # More logging - # Service implementation - use docker-compose-backend-external.yaml if running services in IDE backend: - image: killrvideo/killrvideo-python + build: . ports: - "50101:50101" # Exposes port to be available. depends_on: @@ -33,6 +13,15 @@ services: environment: KILLRVIDEO_LOGGING_LEVEL: debug # More logging + web: # Web Interface + image: killrvideo/killrvideo-web:3 + ports: + - "3000:3000" # Exposes port to be available externally + depends_on: + - backend # Needs Backend to be running + environment: + KILLRVIDEO_LOGGING_LEVEL: debug # More logging + # Datastax Studio provides convenient web interface to work directly with DataStax Enterprise studio: image: datastax/dse-studio:6.7.0 @@ -60,14 +49,14 @@ services: # One-Time Bootstrap Container, configures DSE to have required keyspaces etc. dse-config: - image: killrvideo/killrvideo-dse-config:3.0.0 + image: killrvideo/killrvideo-dse-config:3 depends_on: - dse # Needs DSE to be running # Sample Data Generator, imitates behaviour of users on the killrVideo website. # Adds comments and rates videos, upload new videos and so on. generator: - image: killrvideo/killrvideo-generator:3.0.1 + image: killrvideo/killrvideo-generator:3 depends_on: - backend # Needs Backend to be running environment: