Merge pull request #110 from Duke-MatSci/#89_Setup_Whyis_Docker_Conta…

…iner #89 setup whyis docker container
Duke-MatSci · Apr 5, 2022 · efeadb3 · efeadb3
2 parents c8d5daa + fa47e91
commit efeadb3
Show file tree

Hide file tree

Showing 116 changed files with 63,482 additions and 24 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -91,6 +91,50 @@ services:
     volumes:
       - /app/node_modules
       - ./app:/app
+
+  redis:
+    image: "redis:alpine"
+    command: redis-server
+    volumes:
+     - ./mockDB/redis-data:/var/lib/redis
+  celery:
+    depends_on:
+      - redis
+      - kg
+    volumes:
+      - ./mockDB/fuseki:/app/run
+      - ./whyis/materialsmine:/app
+    command: /opt/venv/bin/celery -A wsgi.celery worker -l INFO -c 4 --uid=nobody --gid=nogroup
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build: whyis
+  celerybeat:
+    depends_on:
+      - redis
+      - kg
+    volumes:
+      - ./mockDB/fuseki:/app/run
+      - ./whyis/materialsmine:/app
+    command: /opt/venv/bin/celery -A wsgi.celery beat -l INFO --uid=nobody --gid=nogroup
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build: whyis
+
+  whyis:
+    depends_on:
+      - redis
+      - kg
+    stdin_open: true
+    command: /opt/venv/bin/gunicorn wsgi:application --workers ${WEB_CONCURRENCY:-8} --timeout 90 -b :8000
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build: whyis
+    volumes:
+      - ./whyis/materialsmine:/app
+      - ./mockDB/fuseki:/app/run
+    ports:
+      - '8000:8000'
+
   kg:
     image: 'stain/jena-fuseki:4.0.0'
     restart: always
@@ -102,4 +146,4 @@ services:
     volumes:
       - ./mockDB/fuseki:/fuseki
 volumes:
-  mockDB:
+  mockDB:
diff --git a/mockDB/fuseki-init.sh b/mockDB/fuseki-init.sh
@@ -0,0 +1 @@
+whyis init
diff --git a/nginx/default.conf b/nginx/default.conf
@@ -12,9 +12,9 @@ upstream minio {
   server minio:9000;
 }
 
-# upstream whyis {
-#   server whyis:5000;
-# }
+upstream whyis {
+  server whyis:8000;
+}
 
 server {
   listen 80;
@@ -39,10 +39,9 @@ server {
     send_timeout 1500;
   }
 
-  # location /whyis {
-  #   rewrite /whyis/(.*) /$1 break;
-  #   proxy_pass http://whyis;
-  # }
+  location /whyis {
+    rewrite /whyis/(.*) /$1 break;
+    proxy_pass http://whyis;
   location /storage {
     sendfile  on;
     keepalive_timeout  60;

diff --git a/whyis/Dockerfile b/whyis/Dockerfile
@@ -1,6 +1,5 @@
-FROM alpine
-RUN apk update && apk add --no-cache python3 py3-pip
-RUN alias python=python3
+FROM tetherlessworld/whyis:latest
+RUN /opt/venv/bin/pip install whyis-unit-converter==0.0.2
+COPY ./materialsmine /app
 WORKDIR '/app'
-COPY . .
-CMD [ "python3" ]
+CMD [ "/bin/bash" ]
diff --git a/whyis/Dockerfile.dev b/whyis/Dockerfile.dev
@@ -1,7 +1,4 @@
-FROM ubuntu:18.04
-RUN apt-get update && apt-get install -y software-properties-common gcc && \
-    add-apt-repository -y ppa:deadsnakes/ppa
-
-RUN apt-get update && apt-get install -y python3 python3-distutils python3-pip python3-apt curl
-WORKDIR '/apps'
-CMD [ "/bin/bash" ]
+FROM tetherlessworld/whyis:latest
+RUN /opt/venv/bin/pip install whyis-unit-converter==0.0.2
+WORKDIR '/app'
+CMD [ "/bin/bash" ]
diff --git a/whyis/docker-compose.yml b/whyis/docker-compose.yml
@@ -1,13 +1,62 @@
 version: '3'
 services:
-  web:
-    stdin_open: true
+  redis:
+    image: "redis:alpine"
+    command: redis-server
+    volumes:
+     - ./mockDB/redis-data:/var/lib/redis
+  celery:
+    depends_on:
+      - redis
+      - fuseki
+    command: /opt/venv/bin/celery -A wsgi.celery worker -l INFO -c 4 --uid=nobody --gid=nogroup
+    volumes:
+      - ./materialsmine:/app
+      - ./mockDB/fuseki:/app/run
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build:
+      dockerfile: Dockerfile.dev
+      context: .
+  celerybeat:
+    depends_on:
+      - redis
+      - fuseki
+    command: /opt/venv/bin/celery -A wsgi.celery beat -l INFO --uid=nobody --gid=nogroup
+    volumes:
+      - ./materialsmine:/app
+      - ./mockDB/fuseki:/app/run
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build:
+      dockerfile: Dockerfile.dev
+      context: .
+  fuseki:
+    command: /opt/venv/bin/whyis fuseki # ; /opt/venv/bin/fuseki-server --mem /ds
     environment:
       - CHOKIDAR_USEPOLLING=true
     build:
+      dockerfile: Dockerfile.dev
       context: .
+    ports:
+      - '3030:3030'
+    volumes:
+      - ./materialsmine:/app
+      - ./mockDB/fuseki:/app/run
+  whyis:
+    depends_on:
+      - redis
+      - fuseki
+    stdin_open: true
+    command: /opt/venv/bin/gunicorn wsgi:application --workers ${WEB_CONCURRENCY:-4} --timeout 90 -b :8000
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    build:
       dockerfile: Dockerfile.dev
+      context: .
+    volumes:
+      - ./materialsmine:/app
+      - ./mockDB/fuseki:/app/run
     ports:
+      - '8000:8000'
       - '5000:5000'
-    volumes:
-      - .:/apps
diff --git a/whyis/materialsmine/.circleci/config.yml b/whyis/materialsmine/.circleci/config.yml
@@ -0,0 +1,29 @@
+version: 2
+jobs:
+  build:
+    working_directory: /apps/nanomine-graph
+    docker:
+      - image: tetherlessworld/whyis:latest
+    steps:
+      - checkout
+      - run:
+          name: Set permissions
+          command: chown -R whyis.whyis /apps/nanomine-graph
+      - run:
+          name: Install application
+          command: pip3 install -e .
+      - run:
+          name: Install newer version of SETLr
+          command: |
+            pip3 uninstall --yes setlr
+            pip3 install -q git+https://github.com/tetherless-world/setlr.git#egg=setlr
+      - run:
+          name: Start the whyis entrypoint (assumes cwd /apps/whyis)
+          command: cd /apps/whyis && /docker-entrypoint.sh
+      - run:
+          name: Run tests
+          command: cd /apps/whyis && python3 manage.py test --ci
+      - store_artifacts:
+          path: /apps/whyis/test-results/py/htmlcov
+      - store_test_results:
+          path: /apps/whyis/test-results
diff --git a/whyis/materialsmine/ChemDatabase/chemprops_example.json b/whyis/materialsmine/ChemDatabase/chemprops_example.json
@@ -0,0 +1,112 @@
+[
+    {
+        "_id": "O(C1=CC=C(C=C1)C(=O)C2=CC=C(OC3=CC=C(C=C3)[*])C=C2)[*]",
+        "_stdname": "Poly(ether ether ketone)",
+        "_abbreviations": [
+            "PEEK"
+        ],
+        "_density": 1.29,
+        "_boc": [
+            "000060020011012102030000100000000000",
+            "000020000010000100000000000000000000"
+        ],
+        "_tradenames": [
+            "Zeniva",
+            "KetaSpire"
+        ],
+        "_synonyms": []
+    },
+    {
+        "_id": " C(C(C1=CC=CC=C1)[*])[*] ",
+        "_stdname": "Polystyrene",
+        "_abbreviations": [
+            "PS"
+        ],
+        "_density": 1.04,
+        "_boc": [
+            "000020000001011101110000200000000000",
+            "000000000000000100100000000000000000",
+            "000040020002021200010000200000000000",
+            "000040020003021200010000300100000000",
+            "010030001002031100000100210000000000"
+        ],
+        "_tradenames": [
+            "Pelaspan",
+            "Afcolene",
+            "Aim",
+            "Amoco",
+            "Bextrene",
+            "Carinex",
+            "Distrene",
+            "Dylene",
+            "Edistir",
+            "Erinoid",
+            "Fina",
+            "Fostarene",
+            "Gedex",
+            "Hostyren",
+            "Huntsman PS",
+            "Lacqrene",
+            "Ladene",
+            "Lorkalene",
+            "Luron",
+            "Lustran",
+            "Lustrex",
+            "Neste PS",
+            "Polystyrol",
+            "Restirolo",
+            "Sicostyrol",
+            "Sternite",
+            "Stiroplasto",
+            "Stymer",
+            "Styrodur",
+            "Styron",
+            "Styvarene",
+            "Vestyron",
+            "Polystyrol",
+            "Polyzote",
+            "Pyrochek 68"
+        ],
+        "_synonyms": [
+            "polyphenylethene",
+            "poly-1-phenylethylene",
+            "polyvinylbenzene"
+        ]
+    },
+    {
+        "_id": "N(CCCCCC(=O)[*])[*]",
+        "_stdname": "Nylon 6",
+        "_abbreviations": [
+            "PA 6",
+            "Nylon 6"
+        ],
+        "_density": 1.08,
+        "_boc": [
+            "000000000001021000000000100000001000",
+            "100000000000000100000000000000001000",
+            "000000000001021000000000100000001000",
+            "302000000002102201010000100000000000",
+            "100110001001101100000000100000001000",
+            "301110002001213201000000100000000000",
+            "303100003001113201000000100000001000",
+            "303100003001113201000000100000000000",
+            "201110001001102201000000100000000000",
+            "302000000002102201010000100000000000",
+            "100040022002224100010002200100000000"
+        ],
+        "_tradenames": [
+            "Perlon",
+            "Plaskon"
+        ],
+        "_synonyms": [
+            "Poly(caprolactam)",
+            "polyamide 6",
+            "poly(ω-aminocaproamide)",
+            "poly(6-aminocaproic acid)",
+            "poly(ε-aminocaproic acid)",
+            "poly(ω-caproamide)",
+            "polycaprolactam",
+            "poly-[imino-(1-oxohexamethylene)]"
+        ]
+    }
+]
diff --git a/whyis/materialsmine/ChemDatabase/notes.md b/whyis/materialsmine/ChemDatabase/notes.md
@@ -0,0 +1,31 @@
+# Chemical Property Database Ingestion
+
+## RDF Structure
+Pure pseudocode:
+```
+chemical
+    id $ID;
+    stdname $STD_NAME;
+    abbreviations $ABBREV_1, $ABBREV_2, ..., $ABBREV_N;
+    density $DENSITY;
+    tradenames $TRADENAME_1, $TRADENAME_2, ..., $TRADENAME_N;
+    synonyms $SYNONYM_1, $SYNONYM_2, ..., $SYNONYM_N;
+```
+
+Applying our ontology:
+```
+PREFIX sio: <http://semanticscience.org/resource/>
+PREFIX nanomine: <http://nanomine.org/ns/>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+
+sio:ChemicalSubstance
+    nanomine:CU_Formula $ID;
+    rdfs:label $STD_NAME;
+    nanomine:Abbreviation $ABBREV_1, $ABBREV_2, ..., $ABBREV_N;
+    sio:Density $DENSITY
+    nanomine:TradeName $TRADENAME_1, $TRADENAME_2, ..., $TRADENAME_N;
+    sio:Synonym $SYNONYM_1, $SYNONYM_2, ..., $SYNONYM_N;
+    
+```
+
+So using this pattern, we have to write an SETLr ingest script to import the data from the JSON files into the knowledge graph.
diff --git a/whyis/materialsmine/LICENSE b/whyis/materialsmine/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Rui Yan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.