Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[MXNET-1155] Add scala packageTest utility #13046

Merged
merged 4 commits into from
Dec 11, 2018
Merged

[MXNET-1155] Add scala packageTest utility #13046

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de65014
[MXNET-1155] Add scala packageTest utility
zachgk Oct 23, 2018
b574992
Clean up utility
zachgk Nov 30, 2018
ea131f0
Safe change directory in Makefile for scala
zachgk Dec 7, 2018
a3c4d45
mvn install file instructions with details
zachgk Dec 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,14 @@ scalaclean:
(cd $(ROOTDIR)/scala-package; \
mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE))

scalatestcompile:
(cd $(ROOTDIR)/scala-package; \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the parenthesis? Also what if cd fails?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mostly left the parenthesis because that is consistent with the rest of the scala make commands and we have been following that since 2452d87. They could all stand to be cleaned up, but that should be handled in a separate PR.

When you say cd fails, are you referring to the change directory or continuous deployment? Either way, is there something that should happen when it fails or some greater consequence of it failing? I'm not sure I get the question

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you might want to use && instead of ; in case the previous command fails.

pllarroy@186590d670bd:0:~$ false ; echo keep going
keep going
pllarroy@186590d670bd:0:~$ false && echo keep going
pllarroy@186590d670bd:1:~$

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://stackoverflow.com/questions/1789594/how-do-i-write-the-cd-command-in-a-makefile

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh. That makes sense. I fixed it

mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalapkg:
(cd $(ROOTDIR)/scala-package; \
mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
Expand Down Expand Up @@ -655,6 +663,15 @@ scaladeploy:
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scaladeploylocal:
(cd $(ROOTDIR)/scala-package; \
mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
-Dgpg.skip \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

jnilint:
3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src

Expand Down
1 change: 1 addition & 0 deletions scala-package/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala
core/src/main/scala/org/apache/mxnet/SymbolBase.scala
examples/scripts/infer/images/
examples/scripts/infer/models/
local-snapshot
14 changes: 14 additions & 0 deletions scala-package/examples/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,20 @@
<platform>linux-x86_64-gpu</platform>
</properties>
</profile>
<profile>
<id>deployLocal</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>release</id>
<build>
Expand Down
87 changes: 87 additions & 0 deletions scala-package/packageTest/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one
zachgk marked this conversation as resolved.
Show resolved Hide resolved
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

SCALA_VERSION_PROFILE := 2.11
SCALA_VERSION := 2.11.8
MXNET_VERSION := "[1.3.0-SNAPSHOT,)"

MXNET_REPO = https://repository.apache.org/content/repositories/snapshots

ifeq ($(OS),Windows_NT)
UNAME_S := Windows
else
UNAME_S := $(shell uname -s)
endif

ifeq ($(UNAME_S), Windows)
# TODO: currently scala package does not support windows
SCALA_PKG_PROFILE := windows
else
ifeq ($(UNAME_S), Darwin)
SCALA_PKG_PROFILE := osx-x86_64-cpu
else
SCALA_PKG_PROFILE := linux-x86_64
ifeq ($(USE_CUDA), 1)
SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu
else
SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu
endif
endif
endif

PROFILES := -Ptest
ifeq ($(UNIT), 1)
PROFILES := "$(PROFILES),unittest"
endif
ifeq ($(INTEGRATION), 1)
PROFILES := "$(PROFILES),integrationtest"
endif

ifneq ($(UNIT), 1)
ifneq ($(INTEGRATION), 1)
PROFILES := "$(PROFILES),unittest,integrationtest"
endif
endif


clean:
(mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testinstall:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES) \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testlocal:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES),fromLocal \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testsnapshot:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES),fromSnapshots \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.repo=$(MXNET_REPO) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))
68 changes: 68 additions & 0 deletions scala-package/packageTest/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# MXNet Scala Package Test

This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile.

## General Setup

To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory:

1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source)
2. Build the tests by running `make scalatestcompile`.
3. Follow setup instructions below for your testing goal

## Running

There are three different modes of operation for testing based on the location of the jar and where it is coming from:

### Test Installed Jars

If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`) using `mvn install:install-file -Dfile=<path-to-file>`. This might be useful if you acquire the .jar file from elsewhere. You can also run `make scalainstall` to install from a local build. Then, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently as @frankfliu tested mvn install:install-file -Dfile=<path-to-file> seemed not functioning properly. Mac OSX crash and Linux Machine installed into a wrong jar. Please double check about this section

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok. I will modify the README instructions to specify either with pom or with all details


zachgk marked this conversation as resolved.
Show resolved Hide resolved
### Test Local Deployment

To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`.

### Remote Repository Snapshot

This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`.

Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`.

### Options

You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above).

An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`.

## Cleaning Up

You can clean temporary files and target artifacts by running `make scalaclean`.

## Troubleshooting

### Missing Examples

If you fail with the following error
```
[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
[ERROR]
[ERROR] After correcting the problems, you can resume the build with the command
[ERROR] mvn <goals> -rf :mxnet-scala-packagetest-examples_2.11
Makefile:57: recipe for target 'scalaintegrationtest' failed
make: *** [scalaintegrationtest] Error 1
```

and stacktrace begins with the following,

```
*** RUN ABORTED ***
java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$
```

you are missing the mxnetexamples package. See your test mode installation section for details.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you link to the test mode installation section so that it is easy to navigate ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In document linking is not officially part of markdown. Github added the feature, but it won't work in other viewers such as intellij. I would rather have no link than a link that might not work

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay! Makes sense

39 changes: 39 additions & 0 deletions scala-package/packageTest/core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>PackageTest</groupId>
<artifactId>mxnet-scala-packagetest_2.11</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>mxnet-scala-packagetest-core_2.11</artifactId>
<name>MXNet Scala Package Test</name>
<packaging>pom</packaging>

<profiles>
<profile>
<id>unittest</id>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>

<build>
<plugins>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<runpath>${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes</runpath>
</configuration>
</plugin>
</plugins>
</build>

</project>
1 change: 1 addition & 0 deletions scala-package/packageTest/core/scripts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../core/scripts
zachgk marked this conversation as resolved.
Show resolved Hide resolved
48 changes: 48 additions & 0 deletions scala-package/packageTest/examples/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>PackageTest</groupId>
<artifactId>mxnet-scala-packagetest_2.11</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>mxnet-scala-packagetest-examples_2.11</artifactId>
<name>MXNet Scala Package Test</name>
<packaging>pom</packaging>

<profiles>
<profile>
<id>integrationtest</id>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>

<build>
<plugins>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<runpath>${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes</runpath>
</configuration>
</plugin>
</plugins>
</build>

<dependencies>
<dependency>
<groupId>org.apache.mxnet</groupId>
<artifactId>mxnet-examples_${mxnet.scalaprofile}</artifactId>
<version>${mxnet.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
1 change: 1 addition & 0 deletions scala-package/packageTest/examples/scripts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../examples/scripts
38 changes: 38 additions & 0 deletions scala-package/packageTest/infer/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>PackageTest</groupId>
<artifactId>mxnet-scala-packagetest_2.11</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>mxnet-scala-packagetest-infer_2.11</artifactId>
<name>MXNet Scala Package Test</name>
<packaging>pom</packaging>

<profiles>
<profile>
<id>unittest</id>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>

<build>
<plugins>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<runpath>${project.build.outputDirectory},${project.build.testOutputDirectory},../../infer/target/test-classes</runpath>
</configuration>
</plugin>
</plugins>
</build>
</project>
Loading