From 9ca3aff6e6802c68cf38392ab01fb52b45a15324 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Tue, 9 May 2023 11:11:50 +0000 Subject: [PATCH 01/10] NER new setup instructions documentation --- .gitignore | 3 ++ dev_setup_examples/.env.example | 62 ++++++++++++++++++++++++ dev_setup_examples/Dockerfile-python3 | 27 +++++++++++ dev_setup_examples/docker-compose.yml | 33 +++++++++++++ dev_setup_examples/entrypoint.sh | 34 ++++++++++++++ docs/install.md | 15 ++++-- docs/ner_new_setup.md | 68 +++++++++++++++++++++++++++ 7 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 dev_setup_examples/.env.example create mode 100644 dev_setup_examples/Dockerfile-python3 create mode 100644 dev_setup_examples/docker-compose.yml create mode 100644 dev_setup_examples/entrypoint.sh create mode 100644 docs/ner_new_setup.md diff --git a/.gitignore b/.gitignore index 20606789..822ecb00 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ logs/*.log* newman_reports/ sftp-config.json dev.json + +# alternate dev setup +!dev_setup_examples/* diff --git a/dev_setup_examples/.env.example b/dev_setup_examples/.env.example new file mode 100644 index 00000000..c50a2f7d --- /dev/null +++ b/dev_setup_examples/.env.example @@ -0,0 +1,62 @@ +# This is .env.example file for chatbot_ner module to hold settings +# Copy it to a docker/.env and fill in all the values. +# Never push your personal keys and passwords to any public repository! +# Please don't add spaces around '=' + +NAME=chatbot_ner +DJANGODIR=/app +ENVIRONMENT=development +DJANGO_LOG_LEVEL=DEBUG +DJANGO_SETTINGS_MODULE=chatbot_ner.settings +DJANGO_WSGI_MODULE=chatbot_ner/wsgi.py +# Important: Change the value of SECRET_KEY to something else and keep it secret +SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c + +NUM_WORKERS=1 +MAX_REQUESTS=1000 +PORT=8081 +TIMEOUT=600 + +# This is the primary engine to use for datastore. Valid values are one of the following: ['elasticsearch'] +ENGINE=elasticsearch + +# ES prefixed variables correspond to settings for elasticsearch. +# ES_URL is the complete url with auth name and password required to connect. If provided, this will override ES_HOST, +# ES_PORT, ES_AUTH_NAME, ES_AUTH_PASSWORD +# ES_HOST and ES_PORT by default is host for ES that comes up with compose + +ES_URL= +ES_AUTH_NAME= +ES_AUTH_PASSWORD= +ES_SCHEME=http +ES_HOST=elasticsearch +ES_PORT=9200 +ES_ALIAS=entity_data +ES_INDEX_1=entity_data_v1 +ES_INDEX_2= +ES_DOC_TYPE=data_dictionary +ELASTICSEARCH_CRF_DATA_INDEX_NAME=entity_examples_data +ELASTICSEARCH_CRF_DATA_DOC_TYPE=training_dictionary + +ES_REQUEST_TIMEOUT=20 +ES_BULK_MSG_SIZE=1000 +ES_SEARCH_SIZE=10000 + +# Auth variables if ES is hosted on AWS +ES_AWS_ACCESS_KEY_ID= +ES_AWS_REGION= +ES_AWS_SECRET_ACCESS_KEY= +ES_AWS_SERVICE= + +DESTINATION_ES_SCHEME= +DESTINATION_HOST= +DESTINATION_PORT= + +# In order to enable entity detection for multiple languages, we use google translate. Please enter the key(optional) +GOOGLE_TRANSLATE_API_KEY= + +# Application performance and error alerting +ELASTIC_APM_ENABLED=False +ELASTIC_APM_SERVER_URL= +SENTRY_ENABLED=False +SENTRY_DSN= \ No newline at end of file diff --git a/dev_setup_examples/Dockerfile-python3 b/dev_setup_examples/Dockerfile-python3 new file mode 100644 index 00000000..0c8bef96 --- /dev/null +++ b/dev_setup_examples/Dockerfile-python3 @@ -0,0 +1,27 @@ +FROM python:3.6.15 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget build-essential curl nginx supervisor && \ + curl -s https://raw.githubusercontent.com/envkey/envkey-source/master/install.sh | bash +WORKDIR /app +COPY requirements.txt nltk_setup.py /app/ +RUN touch /app/config && \ + pip install --no-cache-dir -U pip && \ + pip install --no-cache-dir -U Cython==0.28.2 newrelic==3.4.0.95 && \ + pip install --no-cache-dir -I uwsgi==2.0.19.1 && \ + pip install --no-cache-dir -r /app/requirements.txt && \ + pip check && \ + python /app/nltk_setup.py +ENV NAME="chatbot_ner" +ENV DJANGODIR=/app +ENV NUM_WORKERS=4 +ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings +ENV PORT=8081 +ENV TIMEOUT=600 +ENV MAX_REQUESTS=1000 +# ENV ENVKEY= +# ENV NEWRELIC_LICENSE_KEY= +EXPOSE 8081 +ADD . /app +RUN chmod +x /app/entrypoint.sh +CMD /app/entrypoint.sh diff --git a/dev_setup_examples/docker-compose.yml b/dev_setup_examples/docker-compose.yml new file mode 100644 index 00000000..a3b560cc --- /dev/null +++ b/dev_setup_examples/docker-compose.yml @@ -0,0 +1,33 @@ +version: '3.1' + +services: + chatbot-ner: + image: chatbot_ner:latest + build: + context: . + dockerfile: Dockerfile-python3 + restart: always + env_file: + - .env + volumes: + - .:/app/ + ports: + - 8081:8081 + networks: + - ner + + neres: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + restart: always + environment: + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + volumes: + - ./dbdata/esdata/v1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - ner + +networks: + ner: + driver: bridge diff --git a/dev_setup_examples/entrypoint.sh b/dev_setup_examples/entrypoint.sh new file mode 100644 index 00000000..60b1877a --- /dev/null +++ b/dev_setup_examples/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +export $(envkey-source -f | sed -e 's/\//g' | sed -e 's/\s\+/\n/g' | xargs) > /dev/null +export ENVKEY="" + +cd $DJANGODIR + +export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Create the run directory if it doesn't exist +# RUNDIR=$(dirname $SOCKFILE) +# test -d $RUNDIR || mkdir -p $RUNDIR + + +if [ "$NEWRELIC" == "true" ]; then + + # Define app_name based on the environment to be deployed on. + if [ "$HAPTIK_ENV" == "staging" ]; then + sed -i "s/chatbot_ner/chatbot_ner_staging/g" newrelic.ini + fi + + # Update the license key + sed -i "s/license_value/$NEWRELIC_LICENSE_KEY/g" newrelic.ini + + # Start uwsgi with newrelic agent + export NEW_RELIC_CONFIG_FILE=newrelic.ini + newrelic-admin run-program uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 + +else + + uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600 + +fi diff --git a/docs/install.md b/docs/install.md index 04a7dd15..b0f81fa0 100644 --- a/docs/install.md +++ b/docs/install.md @@ -48,9 +48,12 @@ Following are the steps to create the Docker image and run NER with Docker. 2. **Bring up chatbot_ner:** +> **NOTE** : In case the below chatbot_ner do not work you can always refer to alternate ner setup [here](./ner_new_setup.md) + +**OLD SETUP as below** : ```shell git clone https://github.com/hellohaptik/chatbot_ner.git -cd chatbot_ner +cd chatbot_ner cp .env.example docker/.env # (This will have all the basic environment variables to get started, You can update values accordingly) cd docker docker-compose up --build -d @@ -69,7 +72,7 @@ Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. Now, on your local machine curl the chatbot api as shown shown below, host can be your local machine or a server IP on which you have been running docker on. - > Port mapping can be changed in docker-compose yml + > Port mapping can be changed in docker-compose yml **Container commands:** @@ -78,9 +81,9 @@ cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) -docker-compose restart # (to restart containers, probably when you make code changes) +docker-compose restart # (to restart containers, probably when you make code changes) ``` -**Check logs** +**Check logs** ```shell docker logs -f docker_chatbot-ner_1 ``` @@ -93,13 +96,15 @@ tail -f *.log > > `LOG_LEVEL` can be changed in compose or chatbot_ner/config.py + +## Testing NER API **Example API call to test** Following is an example API call to test our service on your local system/server: ```python entities = ['date','time','restaurant'] -message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" +message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" ``` ```shell diff --git a/docs/ner_new_setup.md b/docs/ner_new_setup.md new file mode 100644 index 00000000..5d68ae0d --- /dev/null +++ b/docs/ner_new_setup.md @@ -0,0 +1,68 @@ +# Alternate Setup for Chatbot ner (Only for Dev purpose with Elasticserach) +Below are steps required for setting up `chatbot_ner` on dev system. Assuming NER setup is at path `~/chatbot_ner` + +**NOTE** : This setup is tested on development environment only therefore its working on other environments is not guareented. + +## Cloning chatbot_ner +If you are doing a new setup. +```bash +git clone https://github.com/hellohaptik/chatbot_ner.git +cd chatbot_ner +``` + +## Stop existing docket setup +In case you are working with existing docker setup and modify it to new one. Then you will have to stop existing NER docker setup. +- Switch to directory `docker/` and stop any existing docker setup running through that directory. + +```bash +cd ~/chatbot_ner/docker +docker-compose down +``` + +- Copy the files from directory `dev_setup_examples` to parent directory. This directory contain examples files which can be configured and used in our new setup. +```bash +cd ~/chatbot_ner/ +cp -r ./dev_setup_examples/* ./ +``` + +## Seting up configuration +These setups should be in parent level directory. Which means inside `~/chatbot_ner/` +- Copy file `.env.example` to `.env` and modify it based on requirements. For updating .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14) . If you don't have access to the notion doc, contact with your manager or someone from ML team. +- Update permission for `entrypoint.sh` file +```bash +sudo chmod 777 entrypoint.sh +``` + +## Running up the container +- Now its time to build the containers and bring them up. +```bash +docker-compose up --build -d +``` +- Verify if containers are up by `docker-compose ps`. Keep monitoring for about 1 minutes to check if elasticsearch container is being restarted. + +- In case elasticsearch container keep restarting. There are some permission issue with the local folder used by elasticsearch container. +There should a folder created by elasticsearch with path `~/chatbot_ner/dbdata/esdata/v1`. +- Now bring down the containers and Provide required permission to above folder. Then bring up the containers. +```bash +cd ~/chatbot_ner + +# stop containers +docker-compose down + +# verify containers are stopped +docker-compose ps + +# give permission to folder +sudo chmod -r 777 ./dbdata/esdata/v1 + +# start the containers +docker-compose up -d +``` + +### Some points to help in case any issue occurs +- check `docker-compose.yml` and make port used there is not already being used by any other service. +- In case any required .env key is missing or setup is not configured propery, containers might restart again and again. Therefore it is advice to keep look on docker logs to check for such errors. + + +## Continue futher setup from Install.md +Now you may need to test the REST API for NER for which refer to section **Testing NER API** in [Install.md](./install.md#testing-ner-api) \ No newline at end of file From 82ec20c7ae0003a3a5f0e19e1286f23f301fcd4e Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Tue, 9 May 2023 12:02:32 +0000 Subject: [PATCH 02/10] updating instruction in existing intall.md document, and fixing grammer issues --- .gitignore | 2 +- .../.env.example | 0 .../Dockerfile-python3 | 0 .../docker-compose.yml | 0 .../entrypoint.sh | 0 docs/install.md | 73 ++++++++++++++----- docs/ner_new_setup.md | 68 ----------------- 7 files changed, 55 insertions(+), 88 deletions(-) rename {dev_setup_examples => dev_setup}/.env.example (100%) rename {dev_setup_examples => dev_setup}/Dockerfile-python3 (100%) rename {dev_setup_examples => dev_setup}/docker-compose.yml (100%) rename {dev_setup_examples => dev_setup}/entrypoint.sh (100%) delete mode 100644 docs/ner_new_setup.md diff --git a/.gitignore b/.gitignore index 822ecb00..4ddf5fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,4 @@ sftp-config.json dev.json # alternate dev setup -!dev_setup_examples/* +!dev_setup/* diff --git a/dev_setup_examples/.env.example b/dev_setup/.env.example similarity index 100% rename from dev_setup_examples/.env.example rename to dev_setup/.env.example diff --git a/dev_setup_examples/Dockerfile-python3 b/dev_setup/Dockerfile-python3 similarity index 100% rename from dev_setup_examples/Dockerfile-python3 rename to dev_setup/Dockerfile-python3 diff --git a/dev_setup_examples/docker-compose.yml b/dev_setup/docker-compose.yml similarity index 100% rename from dev_setup_examples/docker-compose.yml rename to dev_setup/docker-compose.yml diff --git a/dev_setup_examples/entrypoint.sh b/dev_setup/entrypoint.sh similarity index 100% rename from dev_setup_examples/entrypoint.sh rename to dev_setup/entrypoint.sh diff --git a/docs/install.md b/docs/install.md index b0f81fa0..55eae094 100644 --- a/docs/install.md +++ b/docs/install.md @@ -46,30 +46,62 @@ Following are the steps to create the Docker image and run NER with Docker. Please follow the installation document: https://docs.docker.com/docker-for-mac/install/ -2. **Bring up chatbot_ner:** +2. **Cloning chatbot_ner :** + ```bash + git clone https://github.com/hellohaptik/chatbot_ner.git + cd chatbot_ner + ``` + - Copy the files from directory `dev_setup` to parent directory. This directory contains example files which can be configured and used in our new setup. + ```bash + cd ~/chatbot_ner/ + cp -r ./dev_setup/* ./ + ``` + +3. **Seting up configuration:** + Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/ + - (You should change the SECRET_KEY). + - Copy file .env.example to .env and modify it based on requirements. + - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here]([here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14)). If you don't have access to the notion doc, contact your manager or someone from the ML team. + - Update permission for `entrypoint.sh` file + ```bash + sudo chmod 777 entrypoint.sh + ``` + +3. **Bring up chatbot_ner:** + - Now its time to build the containers and bring them up. + ```bash + docker-compose up --build -d + ``` + - Verify if containers are up by docker-compose ps. Keep monitoring for about 1 minute to check if the elasticsearch container is being restarted. + + - In case the elasticsearch container keeps restarting. There are some permission issues with the local folder used by the elasticsearch container. There should be a folder created by elasticsearch with path ~/chatbot_ner/dbdata/esdata/v1. + + - Now bring down the containers and Provide required permission to the above folder. Then bring up the containers. + ```bash + cd ~/chatbot_ner + + # stop containers + docker-compose down + + # verify containers are stopped + docker-compose ps + + # give permission to folder + sudo chmod -r 777 ./dbdata/esdata/v1 + + # start the containers + docker-compose up --build -d + ``` + +The above will also mount the local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. -> **NOTE** : In case the below chatbot_ner do not work you can always refer to alternate ner setup [here](./ner_new_setup.md) - -**OLD SETUP as below** : -```shell -git clone https://github.com/hellohaptik/chatbot_ner.git -cd chatbot_ner -cp .env.example docker/.env # (This will have all the basic environment variables to get started, You can update values accordingly) -cd docker -docker-compose up --build -d -``` - -Open `docker/.env` file and edit the environment variables if needed. (You should change the SECRET_KEY). - -The above will also mount local repo root directory inside the containers /app directory. -Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. > **NOTE**: make sure that nothing is running on port 8081 on your server or your local environment. If anything is running on port 8081, you can stop it by running the following command > > `sudo lsof -t -i tcp:8081 -s tcp:listen | sudo xargs kill` > We have mapped port 80 of the docker container to port 8081 of your machine. - Now, on your local machine curl the chatbot api as shown shown below, + Now, on your local machine curl the chatbot api as shown below, host can be your local machine or a server IP on which you have been running docker on. > Port mapping can be changed in docker-compose yml @@ -85,17 +117,20 @@ docker-compose restart # (to restart containers, probably when you make code cha ``` **Check logs** ```shell -docker logs -f docker_chatbot-ner_1 +docker logs -f chatbot_ner_chatbot-ner_1 ``` ```shell cd ~/chatbot_ner/logs tail -f *.log ``` -> `docker_chatbot-ner_1` is the docker container name +> `chatbot_ner_chatbot-ner_1` is the docker container name > > `LOG_LEVEL` can be changed in compose or chatbot_ner/config.py +### Some points to help in case any issue occurs +- check `docker-compose.yml` and make sure the port being used there is not already being used by any other service. +- In case any required .env key is missing or setup is not configured properly, containers might restart again and again. Therefore it is advised to keep looking at docker logs to check for such errors. ## Testing NER API **Example API call to test** diff --git a/docs/ner_new_setup.md b/docs/ner_new_setup.md deleted file mode 100644 index 5d68ae0d..00000000 --- a/docs/ner_new_setup.md +++ /dev/null @@ -1,68 +0,0 @@ -# Alternate Setup for Chatbot ner (Only for Dev purpose with Elasticserach) -Below are steps required for setting up `chatbot_ner` on dev system. Assuming NER setup is at path `~/chatbot_ner` - -**NOTE** : This setup is tested on development environment only therefore its working on other environments is not guareented. - -## Cloning chatbot_ner -If you are doing a new setup. -```bash -git clone https://github.com/hellohaptik/chatbot_ner.git -cd chatbot_ner -``` - -## Stop existing docket setup -In case you are working with existing docker setup and modify it to new one. Then you will have to stop existing NER docker setup. -- Switch to directory `docker/` and stop any existing docker setup running through that directory. - -```bash -cd ~/chatbot_ner/docker -docker-compose down -``` - -- Copy the files from directory `dev_setup_examples` to parent directory. This directory contain examples files which can be configured and used in our new setup. -```bash -cd ~/chatbot_ner/ -cp -r ./dev_setup_examples/* ./ -``` - -## Seting up configuration -These setups should be in parent level directory. Which means inside `~/chatbot_ner/` -- Copy file `.env.example` to `.env` and modify it based on requirements. For updating .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14) . If you don't have access to the notion doc, contact with your manager or someone from ML team. -- Update permission for `entrypoint.sh` file -```bash -sudo chmod 777 entrypoint.sh -``` - -## Running up the container -- Now its time to build the containers and bring them up. -```bash -docker-compose up --build -d -``` -- Verify if containers are up by `docker-compose ps`. Keep monitoring for about 1 minutes to check if elasticsearch container is being restarted. - -- In case elasticsearch container keep restarting. There are some permission issue with the local folder used by elasticsearch container. -There should a folder created by elasticsearch with path `~/chatbot_ner/dbdata/esdata/v1`. -- Now bring down the containers and Provide required permission to above folder. Then bring up the containers. -```bash -cd ~/chatbot_ner - -# stop containers -docker-compose down - -# verify containers are stopped -docker-compose ps - -# give permission to folder -sudo chmod -r 777 ./dbdata/esdata/v1 - -# start the containers -docker-compose up -d -``` - -### Some points to help in case any issue occurs -- check `docker-compose.yml` and make port used there is not already being used by any other service. -- In case any required .env key is missing or setup is not configured propery, containers might restart again and again. Therefore it is advice to keep look on docker logs to check for such errors. - - -## Continue futher setup from Install.md -Now you may need to test the REST API for NER for which refer to section **Testing NER API** in [Install.md](./install.md#testing-ner-api) \ No newline at end of file From d04af7e8f4070f2b9d4429b6a049f318ed4e1bd3 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Wed, 10 May 2023 04:45:12 +0000 Subject: [PATCH 03/10] using docker folder for new files --- .gitignore | 2 +- dev_setup/docker-compose.yml | 33 ---------------- {dev_setup => docker}/.env.example | 0 docker/Dockerfile | 40 -------------------- {dev_setup => docker}/Dockerfile-python3 | 0 docker/cmd.sh | 13 ------- docker/default.site.conf | 22 ----------- docker/docker-compose.yml | 48 +++++++++--------------- {dev_setup => docker}/entrypoint.sh | 0 docker/nginx.conf | 27 ------------- docker/supervisord.conf | 21 ----------- docs/install.md | 6 +-- 12 files changed, 21 insertions(+), 191 deletions(-) delete mode 100644 dev_setup/docker-compose.yml rename {dev_setup => docker}/.env.example (100%) delete mode 100644 docker/Dockerfile rename {dev_setup => docker}/Dockerfile-python3 (100%) delete mode 100755 docker/cmd.sh delete mode 100644 docker/default.site.conf rename {dev_setup => docker}/entrypoint.sh (100%) delete mode 100644 docker/nginx.conf delete mode 100644 docker/supervisord.conf diff --git a/.gitignore b/.gitignore index 4ddf5fa3..3ea4c953 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,4 @@ sftp-config.json dev.json # alternate dev setup -!dev_setup/* +!docker/* diff --git a/dev_setup/docker-compose.yml b/dev_setup/docker-compose.yml deleted file mode 100644 index a3b560cc..00000000 --- a/dev_setup/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: '3.1' - -services: - chatbot-ner: - image: chatbot_ner:latest - build: - context: . - dockerfile: Dockerfile-python3 - restart: always - env_file: - - .env - volumes: - - .:/app/ - ports: - - 8081:8081 - networks: - - ner - - neres: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 - restart: always - environment: - - "ES_JAVA_OPTS=-Xmx512m -Xms512m" - volumes: - - ./dbdata/esdata/v1:/usr/share/elasticsearch/data - ports: - - 9200:9200 - networks: - - ner - -networks: - ner: - driver: bridge diff --git a/dev_setup/.env.example b/docker/.env.example similarity index 100% rename from dev_setup/.env.example rename to docker/.env.example diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 6f43808f..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM python:3.6.15 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y wget build-essential curl nginx supervisor - -WORKDIR /app - -COPY requirements.txt nltk_setup.py /app/ - -RUN touch /app/config && \ - pip install --no-cache-dir -U pip && \ - pip install --no-cache-dir -r /app/requirements.txt && \ - pip check && \ - python /app/nltk_setup.py - -COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -COPY docker/nginx.conf /etc/nginx/nginx.conf -COPY docker/default.site.conf /etc/nginx/sites-available/default - -# TODO: Separate this out to a dev/test docker image -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ - apt-get install nodejs && \ - npm install -g newman && \ - npm install -g newman-reporter-htmlextra && \ - rm -rf /tmp/* ~/.cache/pip /var/lib/apt/lists/* - - -ENV NAME="chatbot_ner" -ENV DJANGODIR=/app -ENV NUM_WORKERS=4 -ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings -ENV PORT=8081 -ENV TIMEOUT=600 -# Important change this via .env (the file copied from .env.example) -ENV SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c - -ADD . /app -EXPOSE 8081 -# entrypoint/cmd script -CMD /app/docker/cmd.sh diff --git a/dev_setup/Dockerfile-python3 b/docker/Dockerfile-python3 similarity index 100% rename from dev_setup/Dockerfile-python3 rename to docker/Dockerfile-python3 diff --git a/docker/cmd.sh b/docker/cmd.sh deleted file mode 100755 index 26547b4b..00000000 --- a/docker/cmd.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -cd $DJANGODIR - -export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -export PYTHONPATH=$DJANGODIR:$PYTHONPATH - -# Initial setup.py - Datastore lines need to be commented for using previously create data -# TODO: Bad design, should not need to comment, move this as a separate step outside of dockerfile -# we sleep to make sure elasticsearch is up by the time we start setting up -sleep 10 -python /app/datastore_setup.py || { echo 'datastore setup failed'; exit 1; } -/usr/bin/supervisord diff --git a/docker/default.site.conf b/docker/default.site.conf deleted file mode 100644 index ebc1ea37..00000000 --- a/docker/default.site.conf +++ /dev/null @@ -1,22 +0,0 @@ -# This is the server block. This file overwrites the /etc/nginx/sites-available/default that is installed by -# default by nginx -server { - listen 80 default_server; - listen [::]:80 default_server; - error_log /var/log/nginx/error.log; - access_log /var/log/nginx/access.log; - root /app; - location / { - proxy_pass_header Server; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - real_ip_recursive on; - real_ip_header X-Forwarded-For; - set_real_ip_from 0.0.0.0/0; - proxy_connect_timeout 300; - proxy_read_timeout 300; - proxy_pass http://127.0.0.1:8081/; - } -} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 55066beb..a3b560cc 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,47 +1,33 @@ -# Will start the application, mount local directory, -# If you don't want to keep env variables in compose, you can keep an env file that can be copied and source while building the image -# Variables used in the Django app itself can also be configured at chatbot_ner/.env (copy .env.example to .env and edit it) -# This will also bring up local Elasticsearch, you could even use your already setup ES -# I have defined a common network for these 2 services so that containers can communicate with each other - -# TODO move Nginx also into docker-compose -# version: '3.1' services: chatbot-ner: + image: chatbot_ner:latest build: - context: .. - dockerfile: docker/Dockerfile -# Vars being used are defined in .env.example and used in settings.py -# ENV vars defined in Dockerfile can be overwritten here before docker-compose up -# just add to .env - env_file: - - .env + context: . + dockerfile: Dockerfile-python3 restart: always + env_file: + - .env volumes: - - ..:/app/ - -# Map port 8081 of host machine to port 80 of container inside which Nginx and chatbot_ner app are running -# Nginx proxy passes to backend Chatbot_ner app running on 8081 -# Host 8081 -> Nginx 80 -> uwsgi 8081 -# This can be run directly by ports "8081:8081" if you don't want to use Nginx - + - .:/app/ ports: - - '8081:80' + - 8081:8081 networks: - - chatbot_ner - depends_on: - - elasticsearch + - ner - # using Docker Registry Elasticsearch image assuming default datastore engine is Elasticsearch - elasticsearch: - image: "elasticsearch:5.6-alpine" + neres: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + restart: always environment: - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + volumes: + - ./dbdata/esdata/v1:/usr/share/elasticsearch/data + ports: + - 9200:9200 networks: - - chatbot_ner + - ner networks: - chatbot_ner: + ner: driver: bridge diff --git a/dev_setup/entrypoint.sh b/docker/entrypoint.sh similarity index 100% rename from dev_setup/entrypoint.sh rename to docker/entrypoint.sh diff --git a/docker/nginx.conf b/docker/nginx.conf deleted file mode 100644 index 1930b305..00000000 --- a/docker/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -# This file goes to /etc/nginx/nginx.conf -user root; -worker_processes auto; -pid /run/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; - -events { - worker_connections 768; - # multi_accept on; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - include /etc/nginx/mime.types; - default_type application/octet-stream; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - gzip on; - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} diff --git a/docker/supervisord.conf b/docker/supervisord.conf deleted file mode 100644 index d39a3815..00000000 --- a/docker/supervisord.conf +++ /dev/null @@ -1,21 +0,0 @@ -[supervisord] -nodaemon=true - -# Fill in values from ENV - -[program:uwsgi] -command=uwsgi --wsgi-file chatbot_ner/wsgi.py --http :%(ENV_PORT)s --strict --need-app --master --workers=%(ENV_NUM_WORKERS)s --threads 2 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=%(ENV_MAX_REQUESTS)s --harakiri=%(ENV_TIMEOUT)s --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -autorestart=true - -[program:nginx] -command=/usr/sbin/nginx -g "daemon off;" -stdout_logfile= /dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -username=root -autorestart=true diff --git a/docs/install.md b/docs/install.md index 55eae094..f7a5d19b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -51,10 +51,10 @@ Following are the steps to create the Docker image and run NER with Docker. git clone https://github.com/hellohaptik/chatbot_ner.git cd chatbot_ner ``` - - Copy the files from directory `dev_setup` to parent directory. This directory contains example files which can be configured and used in our new setup. + - Copy the files from directory `docker` to parent directory. This directory contains example files which can be configured and used in our new setup. ```bash cd ~/chatbot_ner/ - cp -r ./dev_setup/* ./ + cp -r ./docker/* ./ ``` 3. **Seting up configuration:** @@ -109,7 +109,7 @@ The above will also mount the local repo root directory inside the containers /a **Container commands:** ```shell -cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) +cd ~/chatbot_ner/ # (all compose commands from main directory of the repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) From da1d5b58751a8158f1bce1f424f0c1ea6f5fed16 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Wed, 17 May 2023 04:02:40 +0000 Subject: [PATCH 04/10] Revert "using docker folder for new files" This reverts commit d04af7e8f4070f2b9d4429b6a049f318ed4e1bd3. --- .gitignore | 2 +- {docker => dev_setup}/.env.example | 0 {docker => dev_setup}/Dockerfile-python3 | 0 dev_setup/docker-compose.yml | 33 ++++++++++++++++ {docker => dev_setup}/entrypoint.sh | 0 docker/Dockerfile | 40 ++++++++++++++++++++ docker/cmd.sh | 13 +++++++ docker/default.site.conf | 22 +++++++++++ docker/docker-compose.yml | 48 +++++++++++++++--------- docker/nginx.conf | 27 +++++++++++++ docker/supervisord.conf | 21 +++++++++++ docs/install.md | 6 +-- 12 files changed, 191 insertions(+), 21 deletions(-) rename {docker => dev_setup}/.env.example (100%) rename {docker => dev_setup}/Dockerfile-python3 (100%) create mode 100644 dev_setup/docker-compose.yml rename {docker => dev_setup}/entrypoint.sh (100%) create mode 100644 docker/Dockerfile create mode 100755 docker/cmd.sh create mode 100644 docker/default.site.conf create mode 100644 docker/nginx.conf create mode 100644 docker/supervisord.conf diff --git a/.gitignore b/.gitignore index 3ea4c953..4ddf5fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,4 @@ sftp-config.json dev.json # alternate dev setup -!docker/* +!dev_setup/* diff --git a/docker/.env.example b/dev_setup/.env.example similarity index 100% rename from docker/.env.example rename to dev_setup/.env.example diff --git a/docker/Dockerfile-python3 b/dev_setup/Dockerfile-python3 similarity index 100% rename from docker/Dockerfile-python3 rename to dev_setup/Dockerfile-python3 diff --git a/dev_setup/docker-compose.yml b/dev_setup/docker-compose.yml new file mode 100644 index 00000000..a3b560cc --- /dev/null +++ b/dev_setup/docker-compose.yml @@ -0,0 +1,33 @@ +version: '3.1' + +services: + chatbot-ner: + image: chatbot_ner:latest + build: + context: . + dockerfile: Dockerfile-python3 + restart: always + env_file: + - .env + volumes: + - .:/app/ + ports: + - 8081:8081 + networks: + - ner + + neres: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + restart: always + environment: + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + volumes: + - ./dbdata/esdata/v1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - ner + +networks: + ner: + driver: bridge diff --git a/docker/entrypoint.sh b/dev_setup/entrypoint.sh similarity index 100% rename from docker/entrypoint.sh rename to dev_setup/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..6f43808f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.6.15 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget build-essential curl nginx supervisor + +WORKDIR /app + +COPY requirements.txt nltk_setup.py /app/ + +RUN touch /app/config && \ + pip install --no-cache-dir -U pip && \ + pip install --no-cache-dir -r /app/requirements.txt && \ + pip check && \ + python /app/nltk_setup.py + +COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY docker/nginx.conf /etc/nginx/nginx.conf +COPY docker/default.site.conf /etc/nginx/sites-available/default + +# TODO: Separate this out to a dev/test docker image +RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ + apt-get install nodejs && \ + npm install -g newman && \ + npm install -g newman-reporter-htmlextra && \ + rm -rf /tmp/* ~/.cache/pip /var/lib/apt/lists/* + + +ENV NAME="chatbot_ner" +ENV DJANGODIR=/app +ENV NUM_WORKERS=4 +ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings +ENV PORT=8081 +ENV TIMEOUT=600 +# Important change this via .env (the file copied from .env.example) +ENV SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c + +ADD . /app +EXPOSE 8081 +# entrypoint/cmd script +CMD /app/docker/cmd.sh diff --git a/docker/cmd.sh b/docker/cmd.sh new file mode 100755 index 00000000..26547b4b --- /dev/null +++ b/docker/cmd.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd $DJANGODIR + +export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Initial setup.py - Datastore lines need to be commented for using previously create data +# TODO: Bad design, should not need to comment, move this as a separate step outside of dockerfile +# we sleep to make sure elasticsearch is up by the time we start setting up +sleep 10 +python /app/datastore_setup.py || { echo 'datastore setup failed'; exit 1; } +/usr/bin/supervisord diff --git a/docker/default.site.conf b/docker/default.site.conf new file mode 100644 index 00000000..ebc1ea37 --- /dev/null +++ b/docker/default.site.conf @@ -0,0 +1,22 @@ +# This is the server block. This file overwrites the /etc/nginx/sites-available/default that is installed by +# default by nginx +server { + listen 80 default_server; + listen [::]:80 default_server; + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log; + root /app; + location / { + proxy_pass_header Server; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + real_ip_recursive on; + real_ip_header X-Forwarded-For; + set_real_ip_from 0.0.0.0/0; + proxy_connect_timeout 300; + proxy_read_timeout 300; + proxy_pass http://127.0.0.1:8081/; + } +} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a3b560cc..55066beb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,33 +1,47 @@ +# Will start the application, mount local directory, +# If you don't want to keep env variables in compose, you can keep an env file that can be copied and source while building the image +# Variables used in the Django app itself can also be configured at chatbot_ner/.env (copy .env.example to .env and edit it) +# This will also bring up local Elasticsearch, you could even use your already setup ES +# I have defined a common network for these 2 services so that containers can communicate with each other + +# TODO move Nginx also into docker-compose +# version: '3.1' services: chatbot-ner: - image: chatbot_ner:latest build: - context: . - dockerfile: Dockerfile-python3 - restart: always + context: .. + dockerfile: docker/Dockerfile +# Vars being used are defined in .env.example and used in settings.py +# ENV vars defined in Dockerfile can be overwritten here before docker-compose up +# just add to .env env_file: - - .env + - .env + restart: always volumes: - - .:/app/ + - ..:/app/ + +# Map port 8081 of host machine to port 80 of container inside which Nginx and chatbot_ner app are running +# Nginx proxy passes to backend Chatbot_ner app running on 8081 +# Host 8081 -> Nginx 80 -> uwsgi 8081 +# This can be run directly by ports "8081:8081" if you don't want to use Nginx + ports: - - 8081:8081 + - '8081:80' networks: - - ner + - chatbot_ner + depends_on: + - elasticsearch - neres: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 - restart: always + # using Docker Registry Elasticsearch image assuming default datastore engine is Elasticsearch + elasticsearch: + image: "elasticsearch:5.6-alpine" environment: - "ES_JAVA_OPTS=-Xmx512m -Xms512m" - volumes: - - ./dbdata/esdata/v1:/usr/share/elasticsearch/data - ports: - - 9200:9200 networks: - - ner + - chatbot_ner networks: - ner: + chatbot_ner: driver: bridge diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 00000000..1930b305 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,27 @@ +# This file goes to /etc/nginx/nginx.conf +user root; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + include /etc/nginx/mime.types; + default_type application/octet-stream; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + gzip on; + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 00000000..d39a3815 --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,21 @@ +[supervisord] +nodaemon=true + +# Fill in values from ENV + +[program:uwsgi] +command=uwsgi --wsgi-file chatbot_ner/wsgi.py --http :%(ENV_PORT)s --strict --need-app --master --workers=%(ENV_NUM_WORKERS)s --threads 2 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=%(ENV_MAX_REQUESTS)s --harakiri=%(ENV_TIMEOUT)s --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +autorestart=true + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +stdout_logfile= /dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +username=root +autorestart=true diff --git a/docs/install.md b/docs/install.md index f7a5d19b..55eae094 100644 --- a/docs/install.md +++ b/docs/install.md @@ -51,10 +51,10 @@ Following are the steps to create the Docker image and run NER with Docker. git clone https://github.com/hellohaptik/chatbot_ner.git cd chatbot_ner ``` - - Copy the files from directory `docker` to parent directory. This directory contains example files which can be configured and used in our new setup. + - Copy the files from directory `dev_setup` to parent directory. This directory contains example files which can be configured and used in our new setup. ```bash cd ~/chatbot_ner/ - cp -r ./docker/* ./ + cp -r ./dev_setup/* ./ ``` 3. **Seting up configuration:** @@ -109,7 +109,7 @@ The above will also mount the local repo root directory inside the containers /a **Container commands:** ```shell -cd ~/chatbot_ner/ # (all compose commands from main directory of the repo) +cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) From 3c1690622458c75bea8ed815caddbd3247647f30 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Wed, 17 May 2023 04:59:09 +0000 Subject: [PATCH 05/10] moving files to parent directory and exception in gitignore --- .gitignore | 4 +- .../Dockerfile-python3 => Dockerfile-python3 | 2 +- dev_setup/.env.example | 62 ------------------- .../docker-compose.yml => docker-compose.yml | 2 + docs/install.md | 9 +-- dev_setup/entrypoint.sh => entrypoint.sh | 2 +- 6 files changed, 9 insertions(+), 72 deletions(-) rename dev_setup/Dockerfile-python3 => Dockerfile-python3 (97%) delete mode 100644 dev_setup/.env.example rename dev_setup/docker-compose.yml => docker-compose.yml (93%) rename dev_setup/entrypoint.sh => entrypoint.sh (99%) mode change 100644 => 100755 diff --git a/.gitignore b/.gitignore index 4ddf5fa3..3e2e1b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ sftp-config.json dev.json # alternate dev setup -!dev_setup/* +!docker-compose.yml +!Dockerfile-python3 +!entrypoint.sh diff --git a/dev_setup/Dockerfile-python3 b/Dockerfile-python3 similarity index 97% rename from dev_setup/Dockerfile-python3 rename to Dockerfile-python3 index 0c8bef96..645081ce 100644 --- a/dev_setup/Dockerfile-python3 +++ b/Dockerfile-python3 @@ -24,4 +24,4 @@ ENV MAX_REQUESTS=1000 EXPOSE 8081 ADD . /app RUN chmod +x /app/entrypoint.sh -CMD /app/entrypoint.sh +CMD /app/entrypoint.sh \ No newline at end of file diff --git a/dev_setup/.env.example b/dev_setup/.env.example deleted file mode 100644 index c50a2f7d..00000000 --- a/dev_setup/.env.example +++ /dev/null @@ -1,62 +0,0 @@ -# This is .env.example file for chatbot_ner module to hold settings -# Copy it to a docker/.env and fill in all the values. -# Never push your personal keys and passwords to any public repository! -# Please don't add spaces around '=' - -NAME=chatbot_ner -DJANGODIR=/app -ENVIRONMENT=development -DJANGO_LOG_LEVEL=DEBUG -DJANGO_SETTINGS_MODULE=chatbot_ner.settings -DJANGO_WSGI_MODULE=chatbot_ner/wsgi.py -# Important: Change the value of SECRET_KEY to something else and keep it secret -SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c - -NUM_WORKERS=1 -MAX_REQUESTS=1000 -PORT=8081 -TIMEOUT=600 - -# This is the primary engine to use for datastore. Valid values are one of the following: ['elasticsearch'] -ENGINE=elasticsearch - -# ES prefixed variables correspond to settings for elasticsearch. -# ES_URL is the complete url with auth name and password required to connect. If provided, this will override ES_HOST, -# ES_PORT, ES_AUTH_NAME, ES_AUTH_PASSWORD -# ES_HOST and ES_PORT by default is host for ES that comes up with compose - -ES_URL= -ES_AUTH_NAME= -ES_AUTH_PASSWORD= -ES_SCHEME=http -ES_HOST=elasticsearch -ES_PORT=9200 -ES_ALIAS=entity_data -ES_INDEX_1=entity_data_v1 -ES_INDEX_2= -ES_DOC_TYPE=data_dictionary -ELASTICSEARCH_CRF_DATA_INDEX_NAME=entity_examples_data -ELASTICSEARCH_CRF_DATA_DOC_TYPE=training_dictionary - -ES_REQUEST_TIMEOUT=20 -ES_BULK_MSG_SIZE=1000 -ES_SEARCH_SIZE=10000 - -# Auth variables if ES is hosted on AWS -ES_AWS_ACCESS_KEY_ID= -ES_AWS_REGION= -ES_AWS_SECRET_ACCESS_KEY= -ES_AWS_SERVICE= - -DESTINATION_ES_SCHEME= -DESTINATION_HOST= -DESTINATION_PORT= - -# In order to enable entity detection for multiple languages, we use google translate. Please enter the key(optional) -GOOGLE_TRANSLATE_API_KEY= - -# Application performance and error alerting -ELASTIC_APM_ENABLED=False -ELASTIC_APM_SERVER_URL= -SENTRY_ENABLED=False -SENTRY_DSN= \ No newline at end of file diff --git a/dev_setup/docker-compose.yml b/docker-compose.yml similarity index 93% rename from dev_setup/docker-compose.yml rename to docker-compose.yml index a3b560cc..5cf1d5fe 100644 --- a/dev_setup/docker-compose.yml +++ b/docker-compose.yml @@ -16,11 +16,13 @@ services: networks: - ner + neres: image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 restart: always environment: - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + - "xpack.security.enabled=false" volumes: - ./dbdata/esdata/v1:/usr/share/elasticsearch/data ports: diff --git a/docs/install.md b/docs/install.md index 55eae094..2e9d8aa8 100644 --- a/docs/install.md +++ b/docs/install.md @@ -51,17 +51,12 @@ Following are the steps to create the Docker image and run NER with Docker. git clone https://github.com/hellohaptik/chatbot_ner.git cd chatbot_ner ``` - - Copy the files from directory `dev_setup` to parent directory. This directory contains example files which can be configured and used in our new setup. - ```bash - cd ~/chatbot_ner/ - cp -r ./dev_setup/* ./ - ``` 3. **Seting up configuration:** Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/ - - (You should change the SECRET_KEY). - Copy file .env.example to .env and modify it based on requirements. - - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here]([here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14)). If you don't have access to the notion doc, contact your manager or someone from the ML team. + - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14). If you don't have access to the notion doc, contact your manager or someone from the ML team. + - In .env file you should change the `SECRET_KEY`. - Update permission for `entrypoint.sh` file ```bash sudo chmod 777 entrypoint.sh diff --git a/dev_setup/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 similarity index 99% rename from dev_setup/entrypoint.sh rename to entrypoint.sh index 60b1877a..c3a74d33 --- a/dev_setup/entrypoint.sh +++ b/entrypoint.sh @@ -31,4 +31,4 @@ else uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600 -fi +fi \ No newline at end of file From 9c75a5cffcbcea94eceb2724a411de0b32e1f1ba Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Thu, 18 May 2023 10:55:09 +0000 Subject: [PATCH 06/10] upgraded django version, removed unsed pkg ipython --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4937ca0f..c8d1a516 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ pandas==1.0.5 urllib3==1.26.5 requests==2.26.0 requests-aws4auth==0.9 -Django==3.2.13 +Django==3.2.18 elasticsearch==5.5.3 nltk==3.4.5 @@ -51,7 +51,6 @@ nose-exclude==0.5.0 # Other convenience libs, ideally should only be installed in dev flake8==3.4.1 jedi==0.17.2 -ipython==7.16.3 #logging structlog==21.1.0 From c835f30e755b5fd565568b0e7cb939f8bb18ea31 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Thu, 18 May 2023 11:22:36 +0000 Subject: [PATCH 07/10] removing docker folder as setup is moved outside --- docker/Dockerfile | 40 --------------------------------- docker/cmd.sh | 13 ----------- docker/default.site.conf | 22 ------------------ docker/docker-compose.yml | 47 --------------------------------------- docker/nginx.conf | 27 ---------------------- docker/supervisord.conf | 21 ----------------- 6 files changed, 170 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100755 docker/cmd.sh delete mode 100644 docker/default.site.conf delete mode 100644 docker/docker-compose.yml delete mode 100644 docker/nginx.conf delete mode 100644 docker/supervisord.conf diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 6f43808f..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM python:3.6.15 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y wget build-essential curl nginx supervisor - -WORKDIR /app - -COPY requirements.txt nltk_setup.py /app/ - -RUN touch /app/config && \ - pip install --no-cache-dir -U pip && \ - pip install --no-cache-dir -r /app/requirements.txt && \ - pip check && \ - python /app/nltk_setup.py - -COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -COPY docker/nginx.conf /etc/nginx/nginx.conf -COPY docker/default.site.conf /etc/nginx/sites-available/default - -# TODO: Separate this out to a dev/test docker image -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ - apt-get install nodejs && \ - npm install -g newman && \ - npm install -g newman-reporter-htmlextra && \ - rm -rf /tmp/* ~/.cache/pip /var/lib/apt/lists/* - - -ENV NAME="chatbot_ner" -ENV DJANGODIR=/app -ENV NUM_WORKERS=4 -ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings -ENV PORT=8081 -ENV TIMEOUT=600 -# Important change this via .env (the file copied from .env.example) -ENV SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c - -ADD . /app -EXPOSE 8081 -# entrypoint/cmd script -CMD /app/docker/cmd.sh diff --git a/docker/cmd.sh b/docker/cmd.sh deleted file mode 100755 index 26547b4b..00000000 --- a/docker/cmd.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -cd $DJANGODIR - -export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -export PYTHONPATH=$DJANGODIR:$PYTHONPATH - -# Initial setup.py - Datastore lines need to be commented for using previously create data -# TODO: Bad design, should not need to comment, move this as a separate step outside of dockerfile -# we sleep to make sure elasticsearch is up by the time we start setting up -sleep 10 -python /app/datastore_setup.py || { echo 'datastore setup failed'; exit 1; } -/usr/bin/supervisord diff --git a/docker/default.site.conf b/docker/default.site.conf deleted file mode 100644 index ebc1ea37..00000000 --- a/docker/default.site.conf +++ /dev/null @@ -1,22 +0,0 @@ -# This is the server block. This file overwrites the /etc/nginx/sites-available/default that is installed by -# default by nginx -server { - listen 80 default_server; - listen [::]:80 default_server; - error_log /var/log/nginx/error.log; - access_log /var/log/nginx/access.log; - root /app; - location / { - proxy_pass_header Server; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - real_ip_recursive on; - real_ip_header X-Forwarded-For; - set_real_ip_from 0.0.0.0/0; - proxy_connect_timeout 300; - proxy_read_timeout 300; - proxy_pass http://127.0.0.1:8081/; - } -} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 55066beb..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Will start the application, mount local directory, -# If you don't want to keep env variables in compose, you can keep an env file that can be copied and source while building the image -# Variables used in the Django app itself can also be configured at chatbot_ner/.env (copy .env.example to .env and edit it) -# This will also bring up local Elasticsearch, you could even use your already setup ES -# I have defined a common network for these 2 services so that containers can communicate with each other - -# TODO move Nginx also into docker-compose -# -version: '3.1' - -services: - chatbot-ner: - build: - context: .. - dockerfile: docker/Dockerfile -# Vars being used are defined in .env.example and used in settings.py -# ENV vars defined in Dockerfile can be overwritten here before docker-compose up -# just add to .env - env_file: - - .env - restart: always - volumes: - - ..:/app/ - -# Map port 8081 of host machine to port 80 of container inside which Nginx and chatbot_ner app are running -# Nginx proxy passes to backend Chatbot_ner app running on 8081 -# Host 8081 -> Nginx 80 -> uwsgi 8081 -# This can be run directly by ports "8081:8081" if you don't want to use Nginx - - ports: - - '8081:80' - networks: - - chatbot_ner - depends_on: - - elasticsearch - - # using Docker Registry Elasticsearch image assuming default datastore engine is Elasticsearch - elasticsearch: - image: "elasticsearch:5.6-alpine" - environment: - - "ES_JAVA_OPTS=-Xmx512m -Xms512m" - networks: - - chatbot_ner - -networks: - chatbot_ner: - driver: bridge diff --git a/docker/nginx.conf b/docker/nginx.conf deleted file mode 100644 index 1930b305..00000000 --- a/docker/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -# This file goes to /etc/nginx/nginx.conf -user root; -worker_processes auto; -pid /run/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; - -events { - worker_connections 768; - # multi_accept on; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - include /etc/nginx/mime.types; - default_type application/octet-stream; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - gzip on; - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} diff --git a/docker/supervisord.conf b/docker/supervisord.conf deleted file mode 100644 index d39a3815..00000000 --- a/docker/supervisord.conf +++ /dev/null @@ -1,21 +0,0 @@ -[supervisord] -nodaemon=true - -# Fill in values from ENV - -[program:uwsgi] -command=uwsgi --wsgi-file chatbot_ner/wsgi.py --http :%(ENV_PORT)s --strict --need-app --master --workers=%(ENV_NUM_WORKERS)s --threads 2 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=%(ENV_MAX_REQUESTS)s --harakiri=%(ENV_TIMEOUT)s --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -autorestart=true - -[program:nginx] -command=/usr/sbin/nginx -g "daemon off;" -stdout_logfile= /dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -username=root -autorestart=true From b16b0a6b4fe4025760e5875f7adf6f1cd863f022 Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Thu, 18 May 2023 12:24:49 +0000 Subject: [PATCH 08/10] django version update --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c8d1a516..529ce8c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ pandas==1.0.5 urllib3==1.26.5 requests==2.26.0 requests-aws4auth==0.9 -Django==3.2.18 +Django==3.2.19 elasticsearch==5.5.3 nltk==3.4.5 From af3549db575a3c126c3f05c0e480c10b93eee516 Mon Sep 17 00:00:00 2001 From: harjinder7 <108736141+harjinder7@users.noreply.github.com> Date: Thu, 25 May 2023 12:11:46 +0530 Subject: [PATCH 09/10] Revert "NER new setup instructions documentation" --- .gitignore | 5 --- Dockerfile-python3 | 27 -------------- docker-compose.yml | 35 ------------------ docker/Dockerfile | 40 +++++++++++++++++++++ docker/cmd.sh | 13 +++++++ docker/default.site.conf | 22 ++++++++++++ docker/docker-compose.yml | 47 ++++++++++++++++++++++++ docker/nginx.conf | 27 ++++++++++++++ docker/supervisord.conf | 21 +++++++++++ docs/install.md | 75 +++++++++++---------------------------- entrypoint.sh | 34 ------------------ 11 files changed, 190 insertions(+), 156 deletions(-) delete mode 100644 Dockerfile-python3 delete mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile create mode 100755 docker/cmd.sh create mode 100644 docker/default.site.conf create mode 100644 docker/docker-compose.yml create mode 100644 docker/nginx.conf create mode 100644 docker/supervisord.conf delete mode 100755 entrypoint.sh diff --git a/.gitignore b/.gitignore index 3e2e1b2b..20606789 100644 --- a/.gitignore +++ b/.gitignore @@ -107,8 +107,3 @@ logs/*.log* newman_reports/ sftp-config.json dev.json - -# alternate dev setup -!docker-compose.yml -!Dockerfile-python3 -!entrypoint.sh diff --git a/Dockerfile-python3 b/Dockerfile-python3 deleted file mode 100644 index 645081ce..00000000 --- a/Dockerfile-python3 +++ /dev/null @@ -1,27 +0,0 @@ -FROM python:3.6.15 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y wget build-essential curl nginx supervisor && \ - curl -s https://raw.githubusercontent.com/envkey/envkey-source/master/install.sh | bash -WORKDIR /app -COPY requirements.txt nltk_setup.py /app/ -RUN touch /app/config && \ - pip install --no-cache-dir -U pip && \ - pip install --no-cache-dir -U Cython==0.28.2 newrelic==3.4.0.95 && \ - pip install --no-cache-dir -I uwsgi==2.0.19.1 && \ - pip install --no-cache-dir -r /app/requirements.txt && \ - pip check && \ - python /app/nltk_setup.py -ENV NAME="chatbot_ner" -ENV DJANGODIR=/app -ENV NUM_WORKERS=4 -ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings -ENV PORT=8081 -ENV TIMEOUT=600 -ENV MAX_REQUESTS=1000 -# ENV ENVKEY= -# ENV NEWRELIC_LICENSE_KEY= -EXPOSE 8081 -ADD . /app -RUN chmod +x /app/entrypoint.sh -CMD /app/entrypoint.sh \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5cf1d5fe..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3.1' - -services: - chatbot-ner: - image: chatbot_ner:latest - build: - context: . - dockerfile: Dockerfile-python3 - restart: always - env_file: - - .env - volumes: - - .:/app/ - ports: - - 8081:8081 - networks: - - ner - - - neres: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 - restart: always - environment: - - "ES_JAVA_OPTS=-Xmx512m -Xms512m" - - "xpack.security.enabled=false" - volumes: - - ./dbdata/esdata/v1:/usr/share/elasticsearch/data - ports: - - 9200:9200 - networks: - - ner - -networks: - ner: - driver: bridge diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..6f43808f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.6.15 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget build-essential curl nginx supervisor + +WORKDIR /app + +COPY requirements.txt nltk_setup.py /app/ + +RUN touch /app/config && \ + pip install --no-cache-dir -U pip && \ + pip install --no-cache-dir -r /app/requirements.txt && \ + pip check && \ + python /app/nltk_setup.py + +COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY docker/nginx.conf /etc/nginx/nginx.conf +COPY docker/default.site.conf /etc/nginx/sites-available/default + +# TODO: Separate this out to a dev/test docker image +RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ + apt-get install nodejs && \ + npm install -g newman && \ + npm install -g newman-reporter-htmlextra && \ + rm -rf /tmp/* ~/.cache/pip /var/lib/apt/lists/* + + +ENV NAME="chatbot_ner" +ENV DJANGODIR=/app +ENV NUM_WORKERS=4 +ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings +ENV PORT=8081 +ENV TIMEOUT=600 +# Important change this via .env (the file copied from .env.example) +ENV SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c + +ADD . /app +EXPOSE 8081 +# entrypoint/cmd script +CMD /app/docker/cmd.sh diff --git a/docker/cmd.sh b/docker/cmd.sh new file mode 100755 index 00000000..26547b4b --- /dev/null +++ b/docker/cmd.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd $DJANGODIR + +export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Initial setup.py - Datastore lines need to be commented for using previously create data +# TODO: Bad design, should not need to comment, move this as a separate step outside of dockerfile +# we sleep to make sure elasticsearch is up by the time we start setting up +sleep 10 +python /app/datastore_setup.py || { echo 'datastore setup failed'; exit 1; } +/usr/bin/supervisord diff --git a/docker/default.site.conf b/docker/default.site.conf new file mode 100644 index 00000000..ebc1ea37 --- /dev/null +++ b/docker/default.site.conf @@ -0,0 +1,22 @@ +# This is the server block. This file overwrites the /etc/nginx/sites-available/default that is installed by +# default by nginx +server { + listen 80 default_server; + listen [::]:80 default_server; + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log; + root /app; + location / { + proxy_pass_header Server; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + real_ip_recursive on; + real_ip_header X-Forwarded-For; + set_real_ip_from 0.0.0.0/0; + proxy_connect_timeout 300; + proxy_read_timeout 300; + proxy_pass http://127.0.0.1:8081/; + } +} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..55066beb --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,47 @@ +# Will start the application, mount local directory, +# If you don't want to keep env variables in compose, you can keep an env file that can be copied and source while building the image +# Variables used in the Django app itself can also be configured at chatbot_ner/.env (copy .env.example to .env and edit it) +# This will also bring up local Elasticsearch, you could even use your already setup ES +# I have defined a common network for these 2 services so that containers can communicate with each other + +# TODO move Nginx also into docker-compose +# +version: '3.1' + +services: + chatbot-ner: + build: + context: .. + dockerfile: docker/Dockerfile +# Vars being used are defined in .env.example and used in settings.py +# ENV vars defined in Dockerfile can be overwritten here before docker-compose up +# just add to .env + env_file: + - .env + restart: always + volumes: + - ..:/app/ + +# Map port 8081 of host machine to port 80 of container inside which Nginx and chatbot_ner app are running +# Nginx proxy passes to backend Chatbot_ner app running on 8081 +# Host 8081 -> Nginx 80 -> uwsgi 8081 +# This can be run directly by ports "8081:8081" if you don't want to use Nginx + + ports: + - '8081:80' + networks: + - chatbot_ner + depends_on: + - elasticsearch + + # using Docker Registry Elasticsearch image assuming default datastore engine is Elasticsearch + elasticsearch: + image: "elasticsearch:5.6-alpine" + environment: + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + networks: + - chatbot_ner + +networks: + chatbot_ner: + driver: bridge diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 00000000..1930b305 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,27 @@ +# This file goes to /etc/nginx/nginx.conf +user root; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + include /etc/nginx/mime.types; + default_type application/octet-stream; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + gzip on; + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 00000000..d39a3815 --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,21 @@ +[supervisord] +nodaemon=true + +# Fill in values from ENV + +[program:uwsgi] +command=uwsgi --wsgi-file chatbot_ner/wsgi.py --http :%(ENV_PORT)s --strict --need-app --master --workers=%(ENV_NUM_WORKERS)s --threads 2 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=%(ENV_MAX_REQUESTS)s --harakiri=%(ENV_TIMEOUT)s --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +autorestart=true + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +stdout_logfile= /dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +username=root +autorestart=true diff --git a/docs/install.md b/docs/install.md index 2e9d8aa8..04a7dd15 100644 --- a/docs/install.md +++ b/docs/install.md @@ -46,60 +46,30 @@ Following are the steps to create the Docker image and run NER with Docker. Please follow the installation document: https://docs.docker.com/docker-for-mac/install/ -2. **Cloning chatbot_ner :** - ```bash - git clone https://github.com/hellohaptik/chatbot_ner.git - cd chatbot_ner - ``` - -3. **Seting up configuration:** - Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/ - - Copy file .env.example to .env and modify it based on requirements. - - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14). If you don't have access to the notion doc, contact your manager or someone from the ML team. - - In .env file you should change the `SECRET_KEY`. - - Update permission for `entrypoint.sh` file - ```bash - sudo chmod 777 entrypoint.sh - ``` - -3. **Bring up chatbot_ner:** - - Now its time to build the containers and bring them up. - ```bash - docker-compose up --build -d - ``` - - Verify if containers are up by docker-compose ps. Keep monitoring for about 1 minute to check if the elasticsearch container is being restarted. - - - In case the elasticsearch container keeps restarting. There are some permission issues with the local folder used by the elasticsearch container. There should be a folder created by elasticsearch with path ~/chatbot_ner/dbdata/esdata/v1. - - - Now bring down the containers and Provide required permission to the above folder. Then bring up the containers. - ```bash - cd ~/chatbot_ner - - # stop containers - docker-compose down - - # verify containers are stopped - docker-compose ps - - # give permission to folder - sudo chmod -r 777 ./dbdata/esdata/v1 - - # start the containers - docker-compose up --build -d - ``` - -The above will also mount the local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. +2. **Bring up chatbot_ner:** +```shell +git clone https://github.com/hellohaptik/chatbot_ner.git +cd chatbot_ner +cp .env.example docker/.env # (This will have all the basic environment variables to get started, You can update values accordingly) +cd docker +docker-compose up --build -d +``` + +Open `docker/.env` file and edit the environment variables if needed. (You should change the SECRET_KEY). + +The above will also mount local repo root directory inside the containers /app directory. +Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. > **NOTE**: make sure that nothing is running on port 8081 on your server or your local environment. If anything is running on port 8081, you can stop it by running the following command > > `sudo lsof -t -i tcp:8081 -s tcp:listen | sudo xargs kill` > We have mapped port 80 of the docker container to port 8081 of your machine. - Now, on your local machine curl the chatbot api as shown below, + Now, on your local machine curl the chatbot api as shown shown below, host can be your local machine or a server IP on which you have been running docker on. - > Port mapping can be changed in docker-compose yml + > Port mapping can be changed in docker-compose yml **Container commands:** @@ -108,33 +78,28 @@ cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) -docker-compose restart # (to restart containers, probably when you make code changes) +docker-compose restart # (to restart containers, probably when you make code changes) ``` -**Check logs** +**Check logs** ```shell -docker logs -f chatbot_ner_chatbot-ner_1 +docker logs -f docker_chatbot-ner_1 ``` ```shell cd ~/chatbot_ner/logs tail -f *.log ``` -> `chatbot_ner_chatbot-ner_1` is the docker container name +> `docker_chatbot-ner_1` is the docker container name > > `LOG_LEVEL` can be changed in compose or chatbot_ner/config.py -### Some points to help in case any issue occurs -- check `docker-compose.yml` and make sure the port being used there is not already being used by any other service. -- In case any required .env key is missing or setup is not configured properly, containers might restart again and again. Therefore it is advised to keep looking at docker logs to check for such errors. - -## Testing NER API **Example API call to test** Following is an example API call to test our service on your local system/server: ```python entities = ['date','time','restaurant'] -message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" +message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" ``` ```shell diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index c3a74d33..00000000 --- a/entrypoint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -export $(envkey-source -f | sed -e 's/\//g' | sed -e 's/\s\+/\n/g' | xargs) > /dev/null -export ENVKEY="" - -cd $DJANGODIR - -export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -export PYTHONPATH=$DJANGODIR:$PYTHONPATH - -# Create the run directory if it doesn't exist -# RUNDIR=$(dirname $SOCKFILE) -# test -d $RUNDIR || mkdir -p $RUNDIR - - -if [ "$NEWRELIC" == "true" ]; then - - # Define app_name based on the environment to be deployed on. - if [ "$HAPTIK_ENV" == "staging" ]; then - sed -i "s/chatbot_ner/chatbot_ner_staging/g" newrelic.ini - fi - - # Update the license key - sed -i "s/license_value/$NEWRELIC_LICENSE_KEY/g" newrelic.ini - - # Start uwsgi with newrelic agent - export NEW_RELIC_CONFIG_FILE=newrelic.ini - newrelic-admin run-program uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 - -else - - uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600 - -fi \ No newline at end of file From f479b1486b311cf739539539a779da006ca389ec Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Thu, 25 May 2023 08:15:50 +0000 Subject: [PATCH 10/10] adding setup files in seperate folder --- .gitignore | 2 ++ dev_docker/Dockerfile-python3 | 27 ++++++++++++++++ dev_docker/docker-compose.yml | 35 ++++++++++++++++++++ dev_docker/entrypoint.sh | 34 ++++++++++++++++++++ docs/install.md | 60 +++++++++++++++++++++++++---------- 5 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 dev_docker/Dockerfile-python3 create mode 100644 dev_docker/docker-compose.yml create mode 100644 dev_docker/entrypoint.sh diff --git a/.gitignore b/.gitignore index 20606789..b49c2ab9 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,5 @@ logs/*.log* newman_reports/ sftp-config.json dev.json + +!/dev_docker/* \ No newline at end of file diff --git a/dev_docker/Dockerfile-python3 b/dev_docker/Dockerfile-python3 new file mode 100644 index 00000000..645081ce --- /dev/null +++ b/dev_docker/Dockerfile-python3 @@ -0,0 +1,27 @@ +FROM python:3.6.15 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget build-essential curl nginx supervisor && \ + curl -s https://raw.githubusercontent.com/envkey/envkey-source/master/install.sh | bash +WORKDIR /app +COPY requirements.txt nltk_setup.py /app/ +RUN touch /app/config && \ + pip install --no-cache-dir -U pip && \ + pip install --no-cache-dir -U Cython==0.28.2 newrelic==3.4.0.95 && \ + pip install --no-cache-dir -I uwsgi==2.0.19.1 && \ + pip install --no-cache-dir -r /app/requirements.txt && \ + pip check && \ + python /app/nltk_setup.py +ENV NAME="chatbot_ner" +ENV DJANGODIR=/app +ENV NUM_WORKERS=4 +ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings +ENV PORT=8081 +ENV TIMEOUT=600 +ENV MAX_REQUESTS=1000 +# ENV ENVKEY= +# ENV NEWRELIC_LICENSE_KEY= +EXPOSE 8081 +ADD . /app +RUN chmod +x /app/entrypoint.sh +CMD /app/entrypoint.sh \ No newline at end of file diff --git a/dev_docker/docker-compose.yml b/dev_docker/docker-compose.yml new file mode 100644 index 00000000..f8878510 --- /dev/null +++ b/dev_docker/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3.1' + +services: + chatbot-ner: + image: chatbot_ner:latest + build: + context: . + dockerfile: Dockerfile-python3 + restart: always + env_file: + - .env + volumes: + - .:/app/ + ports: + - 8081:8081 + networks: + - ner + + + neres: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + restart: always + environment: + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + - "xpack.security.enabled=false" + volumes: + - ./dbdata/esdata/v1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - ner + +networks: + ner: + driver: bridge \ No newline at end of file diff --git a/dev_docker/entrypoint.sh b/dev_docker/entrypoint.sh new file mode 100644 index 00000000..c3a74d33 --- /dev/null +++ b/dev_docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +export $(envkey-source -f | sed -e 's/\//g' | sed -e 's/\s\+/\n/g' | xargs) > /dev/null +export ENVKEY="" + +cd $DJANGODIR + +export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Create the run directory if it doesn't exist +# RUNDIR=$(dirname $SOCKFILE) +# test -d $RUNDIR || mkdir -p $RUNDIR + + +if [ "$NEWRELIC" == "true" ]; then + + # Define app_name based on the environment to be deployed on. + if [ "$HAPTIK_ENV" == "staging" ]; then + sed -i "s/chatbot_ner/chatbot_ner_staging/g" newrelic.ini + fi + + # Update the license key + sed -i "s/license_value/$NEWRELIC_LICENSE_KEY/g" newrelic.ini + + # Start uwsgi with newrelic agent + export NEW_RELIC_CONFIG_FILE=newrelic.ini + newrelic-admin run-program uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 + +else + + uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600 + +fi \ No newline at end of file diff --git a/docs/install.md b/docs/install.md index 04a7dd15..ed7e1c4c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -46,18 +46,46 @@ Following are the steps to create the Docker image and run NER with Docker. Please follow the installation document: https://docs.docker.com/docker-for-mac/install/ -2. **Bring up chatbot_ner:** - -```shell -git clone https://github.com/hellohaptik/chatbot_ner.git -cd chatbot_ner -cp .env.example docker/.env # (This will have all the basic environment variables to get started, You can update values accordingly) -cd docker -docker-compose up --build -d -``` - -Open `docker/.env` file and edit the environment variables if needed. (You should change the SECRET_KEY). - +2. **Cloning chatbot_ner :** + ```bash + git clone https://github.com/hellohaptik/chatbot_ner.git + cd chatbot_ner + # copying dev setup files + cp ./dev_docker/* ./ + ``` + +3. **Seting up configuration:** + Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/ + - Copy file `.env.example` to `.env` and modify it based on requirements. + - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14). If you don't have access to the notion doc, contact your manager or someone from the ML team. + - In `.env` file you should change the `SECRET_KEY`. + - Update permission for `entrypoint.sh` file + ```bash + sudo chmod 777 entrypoint.sh + ``` +4. **Bring up chatbot_ner:** + - Now its time to build the containers and bring them up. + ```bash + docker-compose up --build -d + ``` + - Verify if containers are up by docker-compose ps. Keep monitoring for about 1 minute to check if the elasticsearch container is being restarted. + + - In case the elasticsearch container keeps restarting. There are some permission issues with the local folder used by the elasticsearch container. There should be a folder created by elasticsearch with path ~/chatbot_ner/dbdata/esdata/v1. + + - Now bring down the containers and Provide required permission to the above folder. Then bring up the containers. + ```bash + cd ~/chatbot_ner + # stop containers + docker-compose down + # verify containers are stopped + docker-compose ps + # give permission to folder + sudo chmod -r 777 ./dbdata/esdata/v1 + # start the containers + docker-compose up --build -d + ``` + +The above will also mount the local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. The above will also mount local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. > **NOTE**: make sure that nothing is running on port 8081 on your server or your local environment. @@ -69,7 +97,7 @@ Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. Now, on your local machine curl the chatbot api as shown shown below, host can be your local machine or a server IP on which you have been running docker on. - > Port mapping can be changed in docker-compose yml + > Port mapping can be changed in docker-compose yml **Container commands:** @@ -78,9 +106,9 @@ cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) -docker-compose restart # (to restart containers, probably when you make code changes) +docker-compose restart # (to restart containers, probably when you make code changes) ``` -**Check logs** +**Check logs** ```shell docker logs -f docker_chatbot-ner_1 ``` @@ -99,7 +127,7 @@ tail -f *.log ```python entities = ['date','time','restaurant'] -message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" +message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" ``` ```shell