CI/CD improvements (#39)

- Adds an end-to-end test that depends upon `grakn-kglib` from the PyPi test server - Splits out example from main source -Updates READMEs, now there is one for the KGCN example, one for KGCN general usage, and one for the whole repo - Updates some example schema used for the Vienna meetup 29/01/2019 Addresses #35 #36 #37 #31 #30
typedb · Feb 4, 2019 · e0907a0 · e0907a0
1 parent f171590
commit e0907a0
Show file tree

Hide file tree

Showing 21 changed files with 527 additions and 492 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,7 +23,7 @@ jobs:
       - run: unzip grakn-core-1.4.3.zip
       - run: nohup grakn-core-1.4.3/grakn server start
       - run: grakn-core-1.4.3/graql console -k test_schema -f  kglib/kgcn/test_data/schema.gql
-      - run: bazel test //... --test_output=streamed --force_python PY3 --python_path $(which python)
+      - run: bazel test //kglib/... --test_output=streamed --force_python PY3 --python_path $(which python)
 
   test-deploy-pip:
     machine: true
@@ -37,7 +37,17 @@ jobs:
       - run: date +%s > VERSION
       - run: cat VERSION
       - run: bazel run //:deploy-pip -- test $PYPI_TEST_SERVER_USERNAME $PYPI_TEST_SERVER_PASSWORD
-      - run: pip install --extra-index-url https://test.pypi.org/simple/ grakn-kglib
+
+  end-to-end-test:
+    machine: true
+    working_directory: ~/kglib
+    steps:
+      - checkout
+      - bazel_install
+      - run: sudo apt-get update
+      - run: pyenv install 3.6.3
+      - run: pyenv global 3.6.3
+      - run: bazel test //examples:test_pypi_end_to_end_test --test_output=streamed --force_python PY3 --python_path $(which python) --spawn_strategy=standalone
 
   deploy-git:
     machine: true
@@ -71,10 +81,13 @@ workflows:
       - test-deploy-pip:
           requires:
             - test
+      - end-to-end-test:
+          requires:
+            - test-deploy-pip
       - approve-deploy-git:
           type: approval
           requires:
-            - test-deploy-pip
+            - end-to-end-test
       - deploy-git:
           requires:
             - approve-deploy-git

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,9 @@
 # Bazel files
 bazel-*
 
+# IntelliJ Bazel project
+.ijwb/
+
 # Compiled class file
 *.class
 

diff --git a/BUILD b/BUILD
@@ -59,154 +59,5 @@ deploy_pip(
         deployment_requirement("webencodings"),
         deployment_requirement("six"),
     ],
-    target = ":kglib"
+    target = "//kglib:kglib"
 )
-
-py_test(
-    name = "ordered_test",
-    srcs = [
-        "kglib/kgcn/neighbourhood/data/sampling/ordered_test.py"
-    ],
-    deps = [
-        "kglib"
-    ],
-)
-
-py_test(
-    name = "random_sampling_test",
-    srcs = [
-        "kglib/kgcn/neighbourhood/data/sampling/random_sampling_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "label_extraction_test",
-    srcs = [
-        "kglib/kgcn/use_cases/attribute_prediction/label_extraction_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "metrics_test",
-    srcs = [
-        "kglib/kgcn/models/metrics_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "tf_hub_test",
-    srcs = [
-        "kglib/kgcn/encoder/tf_hub_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "schema_test",
-    srcs = [
-        "kglib/kgcn/encoder/schema_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "encode_test",
-    srcs = [
-        "kglib/kgcn/encoder/encode_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "boolean_test",
-    srcs = [
-        "kglib/kgcn/encoder/boolean_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "data_traversal_test",
-    main = "traversal_test.py",
-    srcs = [
-        "kglib/kgcn/neighbourhood/data/traversal_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "data_executor_test",
-    main = "executor_test.py",
-    srcs = [
-        "kglib/kgcn/neighbourhood/data/executor_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "schema_traversal_test",
-    main = "traversal_test.py",
-    srcs = [
-        "kglib/kgcn/neighbourhood/schema/traversal_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_test(
-    name = "raw_array_builder_test",
-    srcs = [
-        "kglib/kgcn/preprocess/raw_array_builder_test.py"
-    ],
-    deps = [
-        "kglib",
-    ]
-)
-
-py_library(
-    name = "kglib",
-    srcs = glob(['kglib/__init__.py', 'kglib/kgcn/**/*.py']),
-    deps = [
-        # Grakn deps
-        requirement('grakn'),
-        requirement('grpcio'),
-
-        # TensorFlow deps
-        requirement('tensorflow'),
-        requirement('numpy'),
-        requirement('protobuf'),
-        requirement('six'),
-        requirement('absl-py'),
-        requirement('keras_applications'),
-        requirement('keras_preprocessing'),
-        requirement('gast'),
-        requirement('astor'),
-        requirement('termcolor'),
-
-        requirement('tensorflow-hub'),
-        requirement('scikit-learn'),
-        requirement('scipy'),
-    ]
-)
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 # Research
 This repository is the centre of all research projects conducted at Grakn Labs. In particular, it's focus is on the integration of machine learning with the Grakn knowledge graph.
 
-At present this repo contains one project: [*Knowledge Graph Convolutional Networks* (KGCNs)](/kglib/kgcn).
+At present this repo contains one project: [*Knowledge Graph Convolutional Networks* (KGCNs)](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn).
 
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.1
+0.1a1
diff --git a/WORKSPACE b/WORKSPACE
@@ -14,7 +14,7 @@ git_repository(
 git_repository(
     name="graknlabs_bazel_distribution",
     remote="https://github.com/graknlabs/bazel-distribution",
-    commit="2e932a2555d1e43f75c8ee676c926399bd12f240"
+    commit="ebc9ae9e6d4ef0086d1c6731bf6f5f8a8f40b509"
 )
 
 ## Only needed for PIP support:
@@ -32,7 +32,16 @@ pip_install()
 
 pip3_import(
     name = "pypi_deployment_dependencies",
-    requirements = "//:deployment/requirements.txt",
+    requirements = "@graknlabs_bazel_distribution//pip:requirements.txt"
 )
 load("@pypi_deployment_dependencies//:requirements.bzl", "pip_install")
-pip_install()
+pip_install()
+
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+http_file(
+  name = "animaltrade_dist",
+  urls = ["https://github.com/graknlabs/kglib/releases/download/v0.1a1/grakn-animaltrade.zip", # TODO How to update to the latest relase each time?
+  ]
+)
diff --git a/deployment.properties b/deployment.properties
@@ -19,7 +19,7 @@
 github.repository=grakn
 maven.repository-url.snapshot=http://maven.grakn.ai/nexus/content/repositories/snapshots/
 maven.repository-url.release=http://maven.grakn.ai/nexus/content/repositories/releases/
-pip.repository-url.pypi=https://pypi.org/legacy/
+pip.repository-url.pypi=https://upload.pypi.org/legacy/
 pip.repository-url.test=https://test.pypi.org/legacy/
 npm.repository-url=https://registry.npmjs.org/
 maven.packages=common,server,console,protocol,client-java
diff --git a/deployment/requirements.txt b/deployment/requirements.txt
diff --git a/examples/BUILD b/examples/BUILD
@@ -0,0 +1,61 @@
+load("@io_bazel_rules_python//python:python.bzl", "py_library", "py_test")
+load("@pypi_dependencies//:requirements.bzl", "requirement")
+
+py_test(
+    name = "test_pypi_end_to_end_test",
+    main = "end_to_end_test.py",
+    srcs = [
+        "kgcn/animal_trade/test/end_to_end_test.py"
+    ],
+    deps = [
+        "test-pypi-kglib",
+    ],
+    data = [
+        "@animaltrade_dist//file",
+    ]
+)
+
+py_test(
+     name = "local_end_to_end_test",
+     main = "end_to_end_test.py",
+     srcs = [
+         "kgcn/animal_trade/test/end_to_end_test.py"
+     ],
+     deps = [
+         "//kglib:kglib",
+     ],
+     data = [
+         "@animaltrade_dist//file",
+     ]
+)
+
+
+py_library(
+    name = "test-pypi-kglib",
+    srcs = [
+        "kgcn/animal_trade/test/end_to_end_test.py"
+    ],
+    deps = [
+        requirement('grakn-kglib'),
+
+        # Grakn deps
+        requirement('grakn'),
+        requirement('grpcio'),
+
+        # TensorFlow deps
+        requirement('tensorflow'),
+        requirement('numpy'),
+        requirement('protobuf'),
+        requirement('six'),
+        requirement('absl-py'),
+        requirement('keras_applications'),
+        requirement('keras_preprocessing'),
+        requirement('gast'),
+        requirement('astor'),
+        requirement('termcolor'),
+
+        requirement('tensorflow-hub'),
+        requirement('scikit-learn'),
+        requirement('scipy'),
+    ]
+)