Skip to content
This repository has been archived by the owner on Nov 18, 2023. It is now read-only.

Commit

Permalink
CI/CD improvements (#39)
Browse files Browse the repository at this point in the history
- Adds an end-to-end test that depends upon `grakn-kglib` from the PyPi test server
- Splits out example from main source
-Updates READMEs, now there is one for the KGCN example, one for KGCN general usage, and one for the whole repo
- Updates some example schema used for the Vienna meetup 29/01/2019

Addresses #35 #36 #37 #31 #30
  • Loading branch information
jmsfltchr authored and Ganeshwara Herawan Hananda committed Feb 4, 2019
1 parent f171590 commit e0907a0
Show file tree
Hide file tree
Showing 21 changed files with 527 additions and 492 deletions.
19 changes: 16 additions & 3 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- run: unzip grakn-core-1.4.3.zip
- run: nohup grakn-core-1.4.3/grakn server start
- run: grakn-core-1.4.3/graql console -k test_schema -f kglib/kgcn/test_data/schema.gql
- run: bazel test //... --test_output=streamed --force_python PY3 --python_path $(which python)
- run: bazel test //kglib/... --test_output=streamed --force_python PY3 --python_path $(which python)

test-deploy-pip:
machine: true
Expand All @@ -37,7 +37,17 @@ jobs:
- run: date +%s > VERSION
- run: cat VERSION
- run: bazel run //:deploy-pip -- test $PYPI_TEST_SERVER_USERNAME $PYPI_TEST_SERVER_PASSWORD
- run: pip install --extra-index-url https://test.pypi.org/simple/ grakn-kglib

end-to-end-test:
machine: true
working_directory: ~/kglib
steps:
- checkout
- bazel_install
- run: sudo apt-get update
- run: pyenv install 3.6.3
- run: pyenv global 3.6.3
- run: bazel test //examples:test_pypi_end_to_end_test --test_output=streamed --force_python PY3 --python_path $(which python) --spawn_strategy=standalone

deploy-git:
machine: true
Expand Down Expand Up @@ -71,10 +81,13 @@ workflows:
- test-deploy-pip:
requires:
- test
- end-to-end-test:
requires:
- test-deploy-pip
- approve-deploy-git:
type: approval
requires:
- test-deploy-pip
- end-to-end-test
- deploy-git:
requires:
- approve-deploy-git
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# Bazel files
bazel-*

# IntelliJ Bazel project
.ijwb/

# Compiled class file
*.class

Expand Down
151 changes: 1 addition & 150 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -59,154 +59,5 @@ deploy_pip(
deployment_requirement("webencodings"),
deployment_requirement("six"),
],
target = ":kglib"
target = "//kglib:kglib"
)

py_test(
name = "ordered_test",
srcs = [
"kglib/kgcn/neighbourhood/data/sampling/ordered_test.py"
],
deps = [
"kglib"
],
)

py_test(
name = "random_sampling_test",
srcs = [
"kglib/kgcn/neighbourhood/data/sampling/random_sampling_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "label_extraction_test",
srcs = [
"kglib/kgcn/use_cases/attribute_prediction/label_extraction_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "metrics_test",
srcs = [
"kglib/kgcn/models/metrics_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "tf_hub_test",
srcs = [
"kglib/kgcn/encoder/tf_hub_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "schema_test",
srcs = [
"kglib/kgcn/encoder/schema_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "encode_test",
srcs = [
"kglib/kgcn/encoder/encode_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "boolean_test",
srcs = [
"kglib/kgcn/encoder/boolean_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "data_traversal_test",
main = "traversal_test.py",
srcs = [
"kglib/kgcn/neighbourhood/data/traversal_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "data_executor_test",
main = "executor_test.py",
srcs = [
"kglib/kgcn/neighbourhood/data/executor_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "schema_traversal_test",
main = "traversal_test.py",
srcs = [
"kglib/kgcn/neighbourhood/schema/traversal_test.py"
],
deps = [
"kglib",
]
)

py_test(
name = "raw_array_builder_test",
srcs = [
"kglib/kgcn/preprocess/raw_array_builder_test.py"
],
deps = [
"kglib",
]
)

py_library(
name = "kglib",
srcs = glob(['kglib/__init__.py', 'kglib/kgcn/**/*.py']),
deps = [
# Grakn deps
requirement('grakn'),
requirement('grpcio'),

# TensorFlow deps
requirement('tensorflow'),
requirement('numpy'),
requirement('protobuf'),
requirement('six'),
requirement('absl-py'),
requirement('keras_applications'),
requirement('keras_preprocessing'),
requirement('gast'),
requirement('astor'),
requirement('termcolor'),

requirement('tensorflow-hub'),
requirement('scikit-learn'),
requirement('scipy'),
]
)
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Research
This repository is the centre of all research projects conducted at Grakn Labs. In particular, it's focus is on the integration of machine learning with the Grakn knowledge graph.

At present this repo contains one project: [*Knowledge Graph Convolutional Networks* (KGCNs)](/kglib/kgcn).
At present this repo contains one project: [*Knowledge Graph Convolutional Networks* (KGCNs)](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn).

2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.1
0.1a1
15 changes: 12 additions & 3 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ git_repository(
git_repository(
name="graknlabs_bazel_distribution",
remote="https://github.com/graknlabs/bazel-distribution",
commit="2e932a2555d1e43f75c8ee676c926399bd12f240"
commit="ebc9ae9e6d4ef0086d1c6731bf6f5f8a8f40b509"
)

## Only needed for PIP support:
Expand All @@ -32,7 +32,16 @@ pip_install()

pip3_import(
name = "pypi_deployment_dependencies",
requirements = "//:deployment/requirements.txt",
requirements = "@graknlabs_bazel_distribution//pip:requirements.txt"
)
load("@pypi_deployment_dependencies//:requirements.bzl", "pip_install")
pip_install()
pip_install()


load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")

http_file(
name = "animaltrade_dist",
urls = ["https://github.com/graknlabs/kglib/releases/download/v0.1a1/grakn-animaltrade.zip", # TODO How to update to the latest relase each time?
]
)
2 changes: 1 addition & 1 deletion deployment.properties
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
github.repository=grakn
maven.repository-url.snapshot=http://maven.grakn.ai/nexus/content/repositories/snapshots/
maven.repository-url.release=http://maven.grakn.ai/nexus/content/repositories/releases/
pip.repository-url.pypi=https://pypi.org/legacy/
pip.repository-url.pypi=https://upload.pypi.org/legacy/
pip.repository-url.test=https://test.pypi.org/legacy/
npm.repository-url=https://registry.npmjs.org/
maven.packages=common,server,console,protocol,client-java
16 changes: 0 additions & 16 deletions deployment/requirements.txt

This file was deleted.

61 changes: 61 additions & 0 deletions examples/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
load("@io_bazel_rules_python//python:python.bzl", "py_library", "py_test")
load("@pypi_dependencies//:requirements.bzl", "requirement")

py_test(
name = "test_pypi_end_to_end_test",
main = "end_to_end_test.py",
srcs = [
"kgcn/animal_trade/test/end_to_end_test.py"
],
deps = [
"test-pypi-kglib",
],
data = [
"@animaltrade_dist//file",
]
)

py_test(
name = "local_end_to_end_test",
main = "end_to_end_test.py",
srcs = [
"kgcn/animal_trade/test/end_to_end_test.py"
],
deps = [
"//kglib:kglib",
],
data = [
"@animaltrade_dist//file",
]
)


py_library(
name = "test-pypi-kglib",
srcs = [
"kgcn/animal_trade/test/end_to_end_test.py"
],
deps = [
requirement('grakn-kglib'),

# Grakn deps
requirement('grakn'),
requirement('grpcio'),

# TensorFlow deps
requirement('tensorflow'),
requirement('numpy'),
requirement('protobuf'),
requirement('six'),
requirement('absl-py'),
requirement('keras_applications'),
requirement('keras_preprocessing'),
requirement('gast'),
requirement('astor'),
requirement('termcolor'),

requirement('tensorflow-hub'),
requirement('scikit-learn'),
requirement('scipy'),
]
)
Loading

0 comments on commit e0907a0

Please sign in to comment.