Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Nightly Tests For Clojure #13624

Merged
merged 1 commit into from
Jan 22, 2019
Merged

Nightly Tests For Clojure #13624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,14 @@ unittest_ubuntu_cpu_clojure() {
./contrib/clojure-package/ci-test.sh
}

unittest_ubuntu_cpu_clojure_integration() {
set -ex
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to inform that we plan to deprecate the make command for Scala package. Please use mvn install instead to cover the Scala portion

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally I'd like to keep the native build, the scala build and the scala packaging separate. Right now, this job is done on demand and this leads to GPU instances compiling code without caching. So this sounds like a move in the right direction!

./contrib/clojure-package/integration-tests.sh
}


unittest_ubuntu_cpugpu_perl() {
set -ex
./perl-package/test.sh
Expand Down
13 changes: 13 additions & 0 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,19 @@ def test_unix_clojure_cpu() {
}]
}

def test_unix_clojure_integration_cpu() {
return ['Clojure: CPU Integration': {
node(NODE_LINUX_CPU) {
gigasquid marked this conversation as resolved.
Show resolved Hide resolved
ws('workspace/ut-clojure-integration-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_lib, true)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure_integration', false)
}
}
}
}]
}

def test_unix_r_cpu() {
return ['R: CPU': {
node(NODE_LINUX_CPU) {
Expand Down
1 change: 1 addition & 0 deletions ci/jenkins/Jenkinsfile_unix_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ core_logic: {
custom_steps.test_unix_scala_cpu(),
custom_steps.test_unix_scala_mkldnn_cpu(),
custom_steps.test_unix_clojure_cpu(),
custom_steps.test_unix_clojure_integration_cpu(),
custom_steps.test_unix_perl_cpu(),
custom_steps.test_unix_r_cpu(),
custom_steps.test_unix_julia07_cpu(),
Expand Down
2 changes: 2 additions & 0 deletions contrib/clojure-package/examples/infer/imageclassifier/src/infer/imageclassifier_example.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@
image-file-batches (->> input-dir
io/file
file-seq
sort
reverse
(filter #(.isFile %))
(filter #(re-matches #".*\.jpg$" (.getPath %)))
(mapv #(.getPath %))
Expand Down
Binary file added contrib/clojure-package/examples/infer/objectdetector/images/marcel.jpg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
32 changes: 22 additions & 10 deletions contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,32 +78,44 @@

(defn detect-single-image
"Detect objects in a single image and print top-5 predictions"
[detector input-image output-dir]
([detector input-dir] (detect-single-image detector input-dir "results"))
([detector input-image output-dir]
(.mkdir (io/file output-dir))
(let [image (infer/load-image-from-file input-image)
topk 5]
topk 5
res (infer/detect-objects detector image topk)
]
(process-results
[input-image]
(infer/detect-objects detector image topk)
output-dir)))
res
output-dir)
(first res)
)))

(defn detect-images-in-dir
"Detect objects in all jpg images in the directory"
[detector input-dir output-dir]
([detector input-dir] (detect-images-in-dir detector input-dir "results"))
([detector input-dir output-dir]
(.mkdir (io/file output-dir))
(let [batch-size 20
image-file-batches (->> input-dir
io/file
file-seq
sort
(filter #(.isFile %))
(filter #(re-matches #".*\.jpg$" (.getPath %)))
(mapv #(.getPath %))
(partition-all batch-size))]
(doall
(apply concat
(for [image-files image-file-batches]
(let [image-batch (infer/load-image-paths image-files) topk 5]
(let [image-batch (infer/load-image-paths image-files)
topk 5
res (infer/detect-objects-batch detector image-batch topk) ]
(process-results
image-files
(infer/detect-objects-batch detector image-batch topk)
output-dir))))))
res
output-dir)
res))))))

(defn run-detector
"Runs an image detector based on options provided"
Expand All @@ -119,7 +131,7 @@
detector (infer/create-object-detector
factory
{:contexts [(context/default-context)]})]
(println "Output results to:" output-dir ":" (.mkdir (io/file output-dir)))
(println "Output results to:" output-dir ":")
(println "Object detection on a single image")
(detect-single-image detector input-image output-dir)
(println "Object detection on images in a directory")
Expand Down
5 changes: 4 additions & 1 deletion .../clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
(let [detector (create-detector)
predictions (detect-single-image detector image-file)
{:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
(clojure.pprint/pprint predictions)
(is (some? predictions))
(is (= 5 (count predictions)))
(is (string? class))
Expand All @@ -55,11 +56,13 @@
(deftest test-batch-detection
(let [detector (create-detector)
batch-predictions (detect-images-in-dir detector image-dir)
_ (clojure.pprint/pprint batch-predictions)
predictions (first batch-predictions)
{:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
(is (some? batch-predictions))
(is (= 5 (count predictions)))
(is (string? class))
(is (< 0.8 prob))
(println [x-min x-max y-min y-max])
(every? #(< 0 % 1) [x-min x-max y-min y-max])
(is (= #{"dog" "person" "bicycle" "car"} (set (mapv :class predictions))))))
(is (= #{"dog" "person"} (set (mapv :class predictions))))))
18 changes: 11 additions & 7 deletions contrib/clojure-package/integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@

set -evx

MXNET_HOME=$(cd "$(dirname $0)/../.."; pwd)
MXNET_HOME=${PWD}
cd ${MXNET_HOME}/contrib/clojure-package
# first build the package and install it
lein install

# then run through the examples
EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples
#cd ${MXNET_HOME}/contrib/clojure-package
#lein test
#lein cloverage --codecov
for test_dir in `find ${EXAMPLES_HOME} -name test` ; do
cd ${test_dir} && lein test
done
# use AWK pattern for blacklisting
TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|dontselect2/'`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice thanks :)

for i in $TEST_CASES ; do
cd ${i} && lein test
done