From d23e9527cc109f9b7325ee8cdec45aed7804c2d6 Mon Sep 17 00:00:00 2001 From: Jan Jansen Date: Sat, 17 Apr 2021 23:24:26 +0200 Subject: [PATCH] Setup testcontainers for hbase [tp-tests] Fixes #1474 * Add docker container for hbase tests * Merge hbase maven projects into one * Update hbase from 2.1.5 to 2.2.7 * Update hadoop from 2.7.7 to 2.8.5 Signed-off-by: Jan Jansen --- .github/workflows/ci-backend-hbase.yml | 26 +- .github/workflows/ci-release.yml | 4 +- .gitignore | 4 - TESTING.md | 14 + docs/changelog.md | 4 +- .../DistributedStoreManagerTest.java | 7 - .../cql/CQLDistributedStoreManagerTest.java | 16 - janusgraph-dist/pom.xml | 10 +- janusgraph-hadoop/pom.xml | 13 + janusgraph-hbase-parent/.gitignore | 10 - .../common/test/binassembly.xml | 15 - .../test/conf/hadoop-metrics.properties | 44 -- .../common/test/conf/hbase-env.sh | 98 ---- .../common/test/conf/hbase-site.xml | 55 --- .../common/test/conf/log4j.properties | 107 ----- .../common/test/conf/regionservers | 1 - .../common/test/resources/mvn_opts.txt | 1 - .../janusgraph-hbase-10/pom.xml | 242 ---------- .../test/java/org/janusgraph/HBaseStatus.java | 117 ----- .../org/janusgraph/HBaseStorageSetup.java | 278 ----------- .../blueprints/HBaseGraphProvider.java | 36 -- .../janusgraph-hbase-server/pom.xml | 143 ------ .../src/test/bin/graceful_stop.sh | 165 ------- .../src/test/bin/hbase | 421 ----------------- .../src/test/bin/hbase-cleanup.sh | 147 ------ .../src/test/bin/hbase-common.sh | 41 -- .../src/test/bin/hbase-config.cmd | 78 ---- .../src/test/bin/hbase-config.sh | 136 ------ .../src/test/bin/hbase-daemon.sh | 284 ----------- .../src/test/bin/hbase-daemons.sh | 53 --- .../src/test/bin/hbase-jruby | 22 - .../src/test/bin/hbase.cmd | 439 ------------------ ...graph_customizations_to_hbase_script.patch | 50 -- .../src/test/bin/local-master-backup.sh | 60 --- .../src/test/bin/local-regionservers.sh | 58 --- .../src/test/bin/master-backup.sh | 74 --- .../src/test/bin/regionservers.sh | 78 ---- .../src/test/bin/rolling-restart.sh | 174 ------- .../src/test/bin/start-hbase.cmd | 61 --- .../src/test/bin/start-hbase.sh | 61 --- .../src/test/bin/stop-hbase.cmd | 54 --- .../src/test/bin/stop-hbase.sh | 65 --- .../test/bin/test/process_based_cluster.sh | 110 ----- .../src/test/bin/zookeepers.sh | 59 --- .../janusgraph-hbase/pom.xml | 172 ------- .../src/assembly/code-jar.xml | 23 - .../src/assembly/javadoc-jar.xml | 23 - .../src/assembly/sources-jar.xml | 23 - janusgraph-hbase-parent/pom.xml | 151 ------ janusgraph-hbase/docker/Dockerfile | 56 +++ .../docker}/hbase-policy.xml | 0 janusgraph-hbase/docker/hbase-site.xml | 33 ++ janusgraph-hbase/docker/zoo.cfg | 15 + .../pom.xml | 101 ++-- .../com/google/common/base/Stopwatch.java | 0 .../java/com/google/common/io/Closeables.java | 0 .../diskstorage/hbase/AdminMask.java | 0 .../diskstorage/hbase/ConnectionMask.java | 0 .../diskstorage/hbase/HBaseAdmin1_0.java | 0 .../diskstorage/hbase/HBaseCompat.java | 0 .../diskstorage/hbase/HBaseCompat1_0.java | 0 .../diskstorage/hbase/HBaseCompatLoader.java | 2 +- .../hbase/HBaseKeyColumnValueStore.java | 0 .../diskstorage/hbase/HBaseStoreManager.java | 0 .../diskstorage/hbase/HBaseTransaction.java | 0 .../diskstorage/hbase/HConnection1_0.java | 0 .../diskstorage/hbase/HTable1_0.java | 0 .../diskstorage/hbase/TableMask.java | 0 .../hadoop/HBaseHadoopStoreManager.java | 0 .../hadoop/HBaseMapReduceIndexJobsUtils.java | 0 .../hadoop/config/HBaseAuthHelper.java | 0 .../formats/hbase/HBaseBinaryInputFormat.java | 0 .../hbase/HBaseBinaryRecordReader.java | 0 .../formats/hbase/HBaseInputFormat.java | 0 .../hbase/HBaseSnapshotBinaryInputFormat.java | 0 .../hbase/HBaseSnapshotInputFormat.java | 0 .../hadoop/scan/HBaseHadoopScanRunner.java | 0 .../java/org/janusgraph/HBaseContainer.java | 176 +++++++ .../HBaseGraphComputerProvider.java | 12 +- .../blueprints/HBaseGraphProvider.java | 66 +++ .../HBaseMultiQueryGraphProvider.java | 11 +- .../blueprints/process/HBaseComputerTest.java | 0 .../process/HBaseMultiQueryProcessTest.java | 0 .../blueprints/process/HBaseProcessTest.java | 0 .../structure/HBaseStructureTest.java | 0 .../HBaseDistributedStoreManagerTest.java | 20 +- .../hbase/HBaseIDAuthorityTest.java | 16 +- .../diskstorage/hbase/HBaseLockStoreTest.java | 17 +- .../diskstorage/hbase/HBaseLogTest.java | 20 +- .../hbase/HBaseMultiWriteStoreTest.java | 18 +- .../hbase/HBaseStoreManagerConfigTest.java | 34 +- .../hbase/HBaseStoreManagerMutationTest.java | 11 +- .../diskstorage/hbase/HBaseStoreTest.java | 19 +- .../hbase/HBaseGraphConcurrentTest.java | 17 +- .../HBaseGraphPerformanceMemoryTest.java | 16 +- .../graphdb/hbase/HBaseGraphTest.java | 16 +- .../graphdb/hbase/HBaseOLAPTest.java | 16 +- .../hbase/HBaseOperationCountingTest.java | 19 +- .../hbase/HBasePartitionGraphTest.java | 17 +- .../hadoop/HBaseIndexManagementIT.java | 24 +- .../janusgraph/hadoop/HBaseInputFormatIT.java | 25 +- .../hadoop/HBaseSnapshotInputFormatIT.java | 67 ++- .../resources/hbase-read-snapshot.properties | 2 + .../src/test/resources/hbase-read.properties | 1 + .../src}/test/resources/log4j.properties | 17 +- mkdocs.yml | 2 +- pom.xml | 8 +- 107 files changed, 644 insertions(+), 4557 deletions(-) delete mode 100644 janusgraph-hbase-parent/.gitignore delete mode 100644 janusgraph-hbase-parent/common/test/binassembly.xml delete mode 100644 janusgraph-hbase-parent/common/test/conf/hadoop-metrics.properties delete mode 100644 janusgraph-hbase-parent/common/test/conf/hbase-env.sh delete mode 100644 janusgraph-hbase-parent/common/test/conf/hbase-site.xml delete mode 100644 janusgraph-hbase-parent/common/test/conf/log4j.properties delete mode 100644 janusgraph-hbase-parent/common/test/conf/regionservers delete mode 100644 janusgraph-hbase-parent/common/test/resources/mvn_opts.txt delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-10/pom.xml delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStatus.java delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-server/pom.xml delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/graceful_stop.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-cleanup.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-common.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.cmd delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemon.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemons.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-jruby delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase.cmd delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/janusgraph_customizations_to_hbase_script.patch delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-master-backup.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-regionservers.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/master-backup.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/regionservers.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/rolling-restart.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.cmd delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.cmd delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/test/process_based_cluster.sh delete mode 100755 janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/zookeepers.sh delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase/pom.xml delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase/src/assembly/code-jar.xml delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase/src/assembly/javadoc-jar.xml delete mode 100644 janusgraph-hbase-parent/janusgraph-hbase/src/assembly/sources-jar.xml delete mode 100644 janusgraph-hbase-parent/pom.xml create mode 100644 janusgraph-hbase/docker/Dockerfile rename {janusgraph-hbase-parent/common/test/conf => janusgraph-hbase/docker}/hbase-policy.xml (100%) create mode 100644 janusgraph-hbase/docker/hbase-site.xml create mode 100644 janusgraph-hbase/docker/zoo.cfg rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/pom.xml (59%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/com/google/common/base/Stopwatch.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/com/google/common/io/Closeables.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/AdminMask.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/ConnectionMask.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-10 => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseAdmin1_0.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-10 => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat1_0.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java (97%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseKeyColumnValueStore.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseStoreManager.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HBaseTransaction.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-10 => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HConnection1_0.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-10 => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/HTable1_0.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/diskstorage/hbase/TableMask.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/HBaseHadoopStoreManager.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/HBaseMapReduceIndexJobsUtils.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/config/HBaseAuthHelper.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryInputFormat.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryRecordReader.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseInputFormat.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotBinaryInputFormat.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotInputFormat.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/main/java/org/janusgraph/hadoop/scan/HBaseHadoopScanRunner.java (100%) create mode 100644 janusgraph-hbase/src/test/java/org/janusgraph/HBaseContainer.java rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java (81%) create mode 100644 janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java (79%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/process/HBaseComputerTest.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/process/HBaseMultiQueryProcessTest.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/process/HBaseProcessTest.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/blueprints/structure/HBaseStructureTest.java (100%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java (77%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java (74%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java (76%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java (74%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java (74%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java (88%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java (94%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java (83%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java (74%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java (75%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java (79%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java (73%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java (74%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java (76%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java (57%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java (80%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java (91%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/resources/hbase-read-snapshot.properties (94%) rename {janusgraph-hbase-parent/janusgraph-hbase-core => janusgraph-hbase}/src/test/resources/hbase-read.properties (97%) rename {janusgraph-hbase-parent/common => janusgraph-hbase/src}/test/resources/log4j.properties (60%) diff --git a/.github/workflows/ci-backend-hbase.yml b/.github/workflows/ci-backend-hbase.yml index 97ae550b2f..8b29d3ff23 100644 --- a/.github/workflows/ci-backend-hbase.yml +++ b/.github/workflows/ci-backend-hbase.yml @@ -60,27 +60,27 @@ jobs: fail-fast: false matrix: include: - - module: hbase-parent/janusgraph-hbase-10 + - module: hbase args: "-Dtest=\"**/diskstorage/hbase/*\"" - name: hbase-10-diskstorage - - module: hbase-parent/janusgraph-hbase-10 + name: hbase1-diskstorage + - module: hbase args: "-Dtest=\"**/graphdb/hbase/*\"" - name: hbase-10-graphdb - - module: hbase-parent/janusgraph-hbase-10 + name: hbase1-graphdb + - module: hbase args: "-Dtest=\"**/hadoop/*\"" - name: hbase-10-hadoop - - module: hbase-parent/janusgraph-hbase-10 + name: hbase1-hadoop + - module: hbase install-args: "-Dhbase.profile -Phbase2" args: "-Dtest=\"**/diskstorage/hbase/*\"" - name: hbase-2-diskstorage - - module: hbase-parent/janusgraph-hbase-10 + name: hbase2-diskstorage + - module: hbase install-args: "-Dhbase.profile -Phbase2" args: "-Dtest=\"**/graphdb/hbase/*\"" - name: hbase-2-graphdb - - module: hbase-parent/janusgraph-hbase-10 + name: hbase2-graphdb + - module: hbase install-args: "-Dhbase.profile -Phbase2" args: "-Dtest=\"**/hadoop/*\"" - name: hbase-2-hadoop + name: hbase2-hadoop steps: - uses: actions/checkout@v2 with: @@ -95,7 +95,7 @@ jobs: with: java-version: 1.8 - run: mvn clean install --projects janusgraph-${{ matrix.module }} ${{ env.BUILD_MAVEN_OPTS }} ${{ matrix.install-args }} - - run: mvn verify --projects janusgraph-${{ matrix.module }} ${{ env.VERIFY_MAVEN_OPTS }} ${{ matrix.args }} + - run: mvn verify --projects janusgraph-${{ matrix.module }} ${{ env.VERIFY_MAVEN_OPTS }} ${{ matrix.args }} -Dhbase.docker.uid=$(id -u) -Dhbase.docker.gid=$(id -g) - uses: actions/upload-artifact@v2 with: name: jacoco-reports diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 02ea5b2738..bf8e0ae4b3 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -94,9 +94,9 @@ jobs: args: "-Pcassandra3-byteordered -Dtest.skip.tp=false -DskipTests=true" - module: cql args: "-Pcassandra3-murmur -Dtest.skip.tp=false -DskipTests=true" - - module: hbase-parent/janusgraph-hbase-10 + - module: hbase args: "-Dtest.skip.tp=false -DskipTests=true" - - module: hbase-parent/janusgraph-hbase-10 + - module: hbase install-args: "-Dhbase.profile -Phbase2" args: "-Dtest.skip.tp=false -DskipTests=true" steps: diff --git a/.gitignore b/.gitignore index abdb00de69..feca7d6c35 100644 --- a/.gitignore +++ b/.gitignore @@ -33,10 +33,6 @@ target/ /janusgraph-examples/example-*/.project /janusgraph-examples/example-*/.settings # End of Eclipse junk -# HBase testing stuff -janusgraph-hbase/logs -janusgraph-hbase/src/test/janusgraph-hbase-test-data/* -janusgraph-hbase/src/test/janusgraph-zookeeper-test/* # Backup files created by `mvn versions:update-child-modules` # http://mojo.codehaus.org/versions-maven-plugin/update-child-modules-mojo.html janusgraph-*/pom.xml.versionsBackup diff --git a/TESTING.md b/TESTING.md index 2c53dc76ae..e468433285 100644 --- a/TESTING.md +++ b/TESTING.md @@ -156,6 +156,20 @@ mvn clean install -pl janusgraph-cql -Dcassandra.docker.image=cassandra mvn clean install -pl janusgraph-cql -Dcassandra.docker.image=cassandra -Dcassandra.docker.version=3.11.2 ``` +### Running hbase tests + +**Note** Running HBase tests require Docker. + +### Special versions of HBase + +System properties to configure HBase test executions: + +| Property | Description | Default value | +| -------- | ----------- | ------------- | +| `hbase.docker.version` | HBase version to be used in the docker image. | `2.2.7` | +| `hbase.docker.uid` | Uid used to run inside HBase of the container | 1000 | +| `hbase.docker.gid` | Gid used to run inside HBase of the container | 1000 | + ### TinkerPop tests The CQL backend is tested with TinkerPop tests using following command. diff --git a/docs/changelog.md b/docs/changelog.md index 8abbc90c76..acb2246618 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -27,7 +27,7 @@ All currently supported verions of JanusGraph are listed below. | JanusGraph | Storage Version | Cassandra | HBase | Bigtable | Elasticsearch | Solr | TinkerPop | Spark | Scala | | ----- | ---- | ---- | ---- | ---- | ---- | ---- | --- | ---- | ---- | | 0.5.z | 2 | 2.1.z, 2.2.z, 3.0.z, 3.11.z | 1.2.z, 1.3.z, 1.4.z, 2.1.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 7.y | 3.4.z | 2.2.z | 2.11.z | -| 0.6.z | 2 | 3.0.z, 3.11.z | 1.2.z, 1.3.z, 1.4.z, 2.1.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 7.y, 8.y | 3.4.z | 2.2.z | 2.11.z | +| 0.6.z | 2 | 3.0.z, 3.11.z | 1.6.z, 2.2.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 7.y, 8.y | 3.4.z | 2.2.z | 2.11.z | #### End-of-Life The versions of JanusGraph listed below are outdated and will no longer receive bugfixes. @@ -58,7 +58,7 @@ compile "org.janusgraph:janusgraph-core:0.6.0" **Tested Compatibility:** * Apache Cassandra 3.0.14, 3.11.10 -* Apache HBase 1.2.6, 1.3.1, 1.4.10, 1.6.0, 2.1.5 +* Apache HBase 1.6.0, 2.2.7 * Google Bigtable 1.3.0, 1.4.0, 1.5.0, 1.6.0, 1.7.0, 1.8.0, 1.9.0, 1.10.0, 1.11.0, 1.14.0 * Oracle BerkeleyJE 7.5.11 * Elasticsearch 6.0.1, 6.6.0, 7.12.0 diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/DistributedStoreManagerTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/DistributedStoreManagerTest.java index 844fddb6c4..b3edcd0e58 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/DistributedStoreManagerTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/diskstorage/DistributedStoreManagerTest.java @@ -16,7 +16,6 @@ import org.janusgraph.JanusGraphBaseStoreFeaturesTest; import org.janusgraph.diskstorage.common.DistributedStoreManager; -import org.janusgraph.diskstorage.common.DistributedStoreManager.Deployment; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore; import org.janusgraph.diskstorage.keycolumnvalue.KeyRange; import org.janusgraph.diskstorage.keycolumnvalue.StoreFeatures; @@ -39,12 +38,6 @@ public StoreFeatures getStoreFeatures(){ return manager.getFeatures(); } - @Test - @FeatureFlag(feature = JanusGraphFeature.OrderedScan) - public void testGetDeployment() { - assertEquals(Deployment.LOCAL, manager.getDeployment()); - } - @Test @FeatureFlag(feature = JanusGraphFeature.OrderedScan) public void testGetLocalKeyPartition() throws BackendException { diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java index 8dfec1cf2a..1af7ebdd1d 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java @@ -17,19 +17,12 @@ import org.janusgraph.JanusGraphCassandraContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.DistributedStoreManagerTest; -import org.janusgraph.diskstorage.common.DistributedStoreManager.Deployment; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; -import org.janusgraph.testutil.FeatureFlag; -import org.janusgraph.testutil.JanusGraphFeature; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import static org.junit.jupiter.api.Assertions.assertEquals; - @Testcontainers public class CQLDistributedStoreManagerTest extends DistributedStoreManagerTest { @@ -51,13 +44,4 @@ public void tearDown() throws BackendException { if (null != manager) manager.close(); } - - @Override - @Test - @FeatureFlag(feature = JanusGraphFeature.OrderedScan) - @Disabled(value = "Test can't run inside of a container") - public void testGetDeployment() { - final Deployment deployment = Deployment.LOCAL; - assertEquals(deployment, manager.getDeployment()); - } } diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml index db81c7b1ee..b7da2047cb 100644 --- a/janusgraph-dist/pom.xml +++ b/janusgraph-dist/pom.xml @@ -69,6 +69,14 @@ com.google.code.gson gson + + org.ow2.asm + asm + + + org.apache.htrace + htrace-core4 + @@ -123,7 +131,7 @@ ${project.groupId} - janusgraph-hbase-core + janusgraph-hbase ${project.version} tests test diff --git a/janusgraph-hadoop/pom.xml b/janusgraph-hadoop/pom.xml index d2716ad080..296be3ec62 100644 --- a/janusgraph-hadoop/pom.xml +++ b/janusgraph-hadoop/pom.xml @@ -91,8 +91,21 @@ org.apache.commons commons-compress + + org.ow2.asm + asm + + + org.apache.htrace + htrace-core4 + + + org.apache.htrace + htrace-core4 + ${htrace.version} + diff --git a/janusgraph-hbase-parent/.gitignore b/janusgraph-hbase-parent/.gitignore deleted file mode 100644 index a37fe6e275..0000000000 --- a/janusgraph-hbase-parent/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -/janusgraph-hbase-*/.settings/ -/janusgraph-hbase-*/.project -/janusgraph-hbase-*/.classpath -/janusgraph-hbase-*/bin/ -/janusgraph-hbase-*/conf/ -/janusgraph-hbase-*/logs/ -*~ -.classpath -.project -.settings/ diff --git a/janusgraph-hbase-parent/common/test/binassembly.xml b/janusgraph-hbase-parent/common/test/binassembly.xml deleted file mode 100644 index d97ecb6c38..0000000000 --- a/janusgraph-hbase-parent/common/test/binassembly.xml +++ /dev/null @@ -1,15 +0,0 @@ - - hbase-bin - / - - - dir - - - - - src/test/bin - / - - - diff --git a/janusgraph-hbase-parent/common/test/conf/hadoop-metrics.properties b/janusgraph-hbase-parent/common/test/conf/hadoop-metrics.properties deleted file mode 100644 index 4cf9478880..0000000000 --- a/janusgraph-hbase-parent/common/test/conf/hadoop-metrics.properties +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019 JanusGraph Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Configuration of the "hbase" context for NullContextWithUpdateThread -# NullContextWithUpdateThread is a null context which has a thread calling -# periodically when monitoring is started. This keeps the data sampled -# correctly. -hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -hbase.period=10 - -# Configuration of the "hbase" context for file -# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# hbase.fileName=/tmp/metrics_hbase.log - -# HBase-specific configuration to reset long-running stats (e.g. compactions) -# If this variable is left out, then the default is no expiration. -hbase.extendedperiod = 3600 - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -jvm.period=10 - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# jvm.fileName=/tmp/metrics_jvm.log - -# Configuration of the "rpc" context for null -rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -rpc.period=10 - -# Configuration of the "rpc" context for file -# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# rpc.fileName=/tmp/metrics_rpc.log \ No newline at end of file diff --git a/janusgraph-hbase-parent/common/test/conf/hbase-env.sh b/janusgraph-hbase-parent/common/test/conf/hbase-env.sh deleted file mode 100644 index c52238f365..0000000000 --- a/janusgraph-hbase-parent/common/test/conf/hbase-env.sh +++ /dev/null @@ -1,98 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -# export JAVA_HOME=/usr/java/jdk1.6.0/ - -for jar in ./target/test-lib/*.jar; do - HBASE_CLASSPATH="$HBASE_CLASSPATH:$jar" -done - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=$HBASE_CLASSPATH - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see https://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC" - -# Uncomment below to enable java garbage collection logging in the .out file. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" - -# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) -# export HBASE_USE_GC_LOGFILE=true - - -# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" -# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. - - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: https://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. -# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -export HBASE_MANAGES_ZK=true diff --git a/janusgraph-hbase-parent/common/test/conf/hbase-site.xml b/janusgraph-hbase-parent/common/test/conf/hbase-site.xml deleted file mode 100644 index b2b2c8c400..0000000000 --- a/janusgraph-hbase-parent/common/test/conf/hbase-site.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - hbase.rootdir - file://$MAVEN{project.build.directory}/hbase-root - - - hbase.zookeeper.property.dataDir - $MAVEN{project.build.directory}/zk-data - - - zookeeper.session.timeout - 1200000 - - - hbase.zookeeper.property.tickTime - 6000 - - - hbase.defaults.for.version.skip - true - - diff --git a/janusgraph-hbase-parent/common/test/conf/log4j.properties b/janusgraph-hbase-parent/common/test/conf/log4j.properties deleted file mode 100644 index 6911772b30..0000000000 --- a/janusgraph-hbase-parent/common/test/conf/log4j.properties +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2019 JanusGraph Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.security.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n - -# Rolling File Appender properties -hbase.log.maxfilesize=256MB -hbase.log.maxbackupindex=20 - -# Rolling File Appender -log4j.appender.RFA=org.apache.log4j.RollingFileAppender -log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} - -log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} -log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} - -log4j.appender.RFA.layout=org.apache.log4j.PatternLayout -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n - -# -# Security audit appender -# -hbase.security.log.file=SecurityAuth.audit -hbase.security.log.maxfilesize=256MB -hbase.security.log.maxbackupindex=20 -log4j.appender.RFAS=org.apache.log4j.RollingFileAppender -log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} -log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} -log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} -log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.category.SecurityLogger=${hbase.security.logger} -log4j.additivity.SecurityLogger=false -#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE - -# -# Null Appender -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n - -# Custom Logging levels - -log4j.logger.org.apache.zookeeper=INFO -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.hbase=DEBUG -# Make these two classes INFO-level. Make them DEBUG to see more zk debug. -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO -#log4j.logger.org.apache.hadoop.dfs=DEBUG -# Set this class to log INFO only otherwise its OTT -# Enable this to get detailed connection error/retry logging. -# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE - - -# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) -#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG - -# Uncomment the below if you want to remove logging of client region caching' -# and scan of .META. messages -# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO -# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/janusgraph-hbase-parent/common/test/conf/regionservers b/janusgraph-hbase-parent/common/test/conf/regionservers deleted file mode 100644 index 7b9ad531d2..0000000000 --- a/janusgraph-hbase-parent/common/test/conf/regionservers +++ /dev/null @@ -1 +0,0 @@ -127.0.0.1 diff --git a/janusgraph-hbase-parent/common/test/resources/mvn_opts.txt b/janusgraph-hbase-parent/common/test/resources/mvn_opts.txt deleted file mode 100644 index 76bb00175c..0000000000 --- a/janusgraph-hbase-parent/common/test/resources/mvn_opts.txt +++ /dev/null @@ -1 +0,0 @@ -${janusgraph.hbase.mvn.opts} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-10/pom.xml b/janusgraph-hbase-parent/janusgraph-hbase-10/pom.xml deleted file mode 100644 index 1098f29995..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-10/pom.xml +++ /dev/null @@ -1,242 +0,0 @@ - - 4.0.0 - - org.janusgraph - janusgraph-hbase-parent - 0.6.0-SNAPSHOT - ../pom.xml - - janusgraph-hbase-10 - JanusGraph-HBase: 1.x and 2.x Compatibility Shim - https://janusgraph.org - - - ${basedir}/../.. - - - - - ${project.groupId} - janusgraph-hbase-core - ${project.version} - - - commons-net - commons-net - - - - - org.janusgraph - janusgraph-backend-testutils - ${project.version} - test - - - ${project.groupId} - janusgraph-hbase-server - ${project.version} - test - - - org.janusgraph - janusgraph-hadoop - ${project.version} - true - - - org.apache.commons - commons-compress - - - - - org.janusgraph - janusgraph-hadoop - ${project.version} - tests - test - - - - - org.slf4j - slf4j-log4j12 - true - - - log4j - log4j - true - - - ch.qos.logback - logback-classic - - - - - src/main/java - ../common/test/java - - - ${project.basedir}/../common/test/resources - true - - - - - maven-dependency-plugin - - - generate-cached-classpath - - - unpack-common-test-classes - - - - - maven-clean-plugin - - - clean-hbase-files - - - - - maven-resources-plugin - - - filter-hbase-conf - none - - - - - maven-deploy-plugin - - - default-deploy - none - - - - - - - - - hbase1 - - - org.apache.hbase - hbase-shaded-client - ${hbase1.version} - - - junit - junit - - - true - - - org.apache.hbase - hbase-shaded-server - ${hbase1.version} - true - - - - - hbase2 - - - !hbase.profile - - - - - org.apache.hbase - hbase-shaded-client - ${hbase2.version} - - - junit - junit - - - true - - - org.apache.hbase - hbase-shaded-mapreduce - ${hbase2.version} - - - junit - junit - - - true - - - - - hadoop2 - - - - !hadoop.profile - - - - - - org.apache.hadoop - hadoop-annotations - ${hadoop2.version} - true - - - org.apache.hadoop - hadoop-hdfs - ${hadoop2.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop2.version} - true - - - com.thoughtworks.paranamer - paranamer - - - - - - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStatus.java b/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStatus.java deleted file mode 100644 index 5ed702f9ea..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStatus.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.janusgraph.util.system.IOUtils; - -public class HBaseStatus { - - private static final Logger log = - LoggerFactory.getLogger(HBaseStatus.class); - - private final File file; - private final String version; - - private HBaseStatus(File file, String version) { - Preconditions.checkNotNull(file); - Preconditions.checkNotNull(version); - this.file = file; - this.version = version; - } - - public String getVersion() { - return version; - } - - public File getFile() { - return file; - } - - public String getScriptDir() { - return HBaseStorageSetup.getScriptDirForHBaseVersion(version); - } - - public String getConfDir() { - return HBaseStorageSetup.getConfDirForHBaseVersion(version); - } - - public static HBaseStatus read(String path) { - - File pid = new File(path); - - if (!pid.exists()) { - log.info("HBase pidfile {} does not exist", path); - return null; - } - - BufferedReader pidReader = null; - try { - pidReader = new BufferedReader(new FileReader(pid)); - HBaseStatus s = parsePidFile(pid, pidReader); - log.info("Read HBase status from {}", path); - return s; - } catch (HBasePidfileParseException | IOException e) { - log.warn("Assuming HBase is not running", e); - } finally { - IOUtils.closeQuietly(pidReader); - } - - return null; - } - - public static HBaseStatus write(String path, String hbaseVersion) { - File f = new File(path); - - FileOutputStream fos = null; - try { - fos = new FileOutputStream(path); - fos.write(String.format("%s", hbaseVersion).getBytes()); - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(fos); - } - - return new HBaseStatus(f, hbaseVersion); - } - - private static HBaseStatus parsePidFile(File f, BufferedReader br) throws HBasePidfileParseException, IOException { - String l = br.readLine(); - - if (null == l || "".equals(l.trim())) { - throw new HBasePidfileParseException("Empty HBase status file " + f); - } - - return new HBaseStatus(f, l.trim()); - } - - private static class HBasePidfileParseException extends Exception { - private static final long serialVersionUID = 1L; - public HBasePidfileParseException(String message) { - super(message); - } - } -} - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java b/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java deleted file mode 100644 index 40e1be9c19..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph; - -import org.janusgraph.diskstorage.BackendException; -import org.janusgraph.diskstorage.TemporaryBackendException; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; -import org.janusgraph.diskstorage.configuration.WriteConfiguration; -import org.janusgraph.diskstorage.hbase.AdminMask; -import org.janusgraph.diskstorage.hbase.ConnectionMask; -import org.janusgraph.diskstorage.hbase.HBaseCompat; -import org.janusgraph.diskstorage.hbase.HBaseCompatLoader; -import org.janusgraph.diskstorage.hbase.HBaseStoreManager; -import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; -import org.janusgraph.graphdb.database.idassigner.placement.SimpleBulkPlacementStrategy; - -import org.apache.commons.lang3.StringUtils; - -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.util.VersionInfo; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class HBaseStorageSetup { - - private static final Logger log = LoggerFactory.getLogger(HBaseStorageSetup.class); - - // hbase config for testing - - public static final String HBASE_PARENT_DIR_PROP = "test.hbase.parentdir"; - - private static final Pattern HBASE_SUPPORTED_VERSION_PATTERN = Pattern.compile("^((2\\.[01])|(1\\.[234]))\\..*"); - - private static final String HBASE_VERSION_1_STRING = "1."; - - private static final String HBASE_PARENT_DIR; - - private static final String HBASE_TARGET_VERSION = VersionInfo.getVersion(); - - private static final HBaseCompat compat; - - static { - String parentDir = ".."; - String tmp = System.getProperty(HBASE_PARENT_DIR_PROP); - if (null != tmp) { - parentDir = tmp; - } - HBASE_PARENT_DIR = parentDir; - compat = HBaseCompatLoader.getCompat(null); - } - - private static final String HBASE_STAT_FILE = "/tmp/janusgraph-hbase-test-daemon.stat"; - - private volatile static HBaseStatus HBASE = null; - - public static String getScriptDirForHBaseVersion(String hv) { - return getDirForHBaseVersion(hv, "bin"); - } - - public static String getConfDirForHBaseVersion(String hv) { - return getDirForHBaseVersion(hv, "conf"); - } - - public static String getDirForHBaseVersion(String hv, String lastSubdirectory) { - Matcher m = HBASE_SUPPORTED_VERSION_PATTERN.matcher(hv); - if (m.matches()) { - String result = String.format("%s%sjanusgraph-hbase-server/%s/", HBASE_PARENT_DIR, File.separator, lastSubdirectory); - log.debug("Built {} path for HBase version {}: {}", lastSubdirectory, hv, result); - return result; - } else { - throw new RuntimeException("Unsupported HBase test version " + hv + " does not match pattern " + HBASE_SUPPORTED_VERSION_PATTERN); - } - } - - public static ModifiableConfiguration getHBaseConfiguration() { - return getHBaseConfiguration(""); - } - - public static ModifiableConfiguration getHBaseConfiguration(String tableName) { - return getHBaseConfiguration(tableName, ""); - } - - public static ModifiableConfiguration getHBaseConfiguration(String tableName, String graphName) { - ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); - config.set(GraphDatabaseConfiguration.STORAGE_BACKEND, "hbase"); - if (!StringUtils.isEmpty(tableName)) config.set(HBaseStoreManager.HBASE_TABLE, tableName); - if (!StringUtils.isEmpty(graphName)) config.set(GraphDatabaseConfiguration.GRAPH_NAME, graphName); - config.set(GraphDatabaseConfiguration.TIMESTAMP_PROVIDER, HBaseStoreManager.PREFERRED_TIMESTAMPS); - config.set(SimpleBulkPlacementStrategy.CONCURRENT_PARTITIONS, 1); - config.set(GraphDatabaseConfiguration.DROP_ON_CLEAR, false); - - return config; - } - - public static WriteConfiguration getHBaseGraphConfiguration() { - return getHBaseConfiguration().getConfiguration(); - } - - /** - * Starts the HBase version described by {@link #HBASE_TARGET_VERSION} - * - * @return a status object describing a successfully-started HBase daemon - * @throws IOException - * passed-through - * @throws RuntimeException - * if starting HBase fails for any other reason - */ - public synchronized static HBaseStatus startHBase() throws IOException { - if (HBASE != null) { - log.info("HBase already started"); - return HBASE; - } - - killIfRunning(); - - deleteData(); - - log.info("Starting HBase"); - String scriptPath = getScriptDirForHBaseVersion(HBASE_TARGET_VERSION) + "/hbase-daemon.sh"; - DaemonRunner.runCommand(scriptPath, "--config", getConfDirForHBaseVersion(HBASE_TARGET_VERSION), "start", "master"); - - HBASE = HBaseStatus.write(HBASE_STAT_FILE, HBASE_TARGET_VERSION); - - registerKillerHook(HBASE); - - waitForConnection(60L, TimeUnit.SECONDS); - - return HBASE; - } - - /** - * Check whether {@link #HBASE_STAT_FILE} describes an HBase daemon. If so, - * kill it. Otherwise, do nothing. - */ - public synchronized static void killIfRunning() { - HBaseStatus stat = HBaseStatus.read(HBASE_STAT_FILE); - - if (null == stat) { - log.info("HBase is not running"); - return; - } - - shutdownHBase(stat); - } - - public synchronized static void waitForConnection(long timeout, TimeUnit timeoutUnit) { - long before = System.currentTimeMillis(); - long after; - long timeoutMS = TimeUnit.MILLISECONDS.convert(timeout, timeoutUnit); - do { - try (ConnectionMask hc = compat.createConnection(HBaseConfiguration.create())) { - after = System.currentTimeMillis(); - log.info("HBase server to started after about {} ms", after - before); - return; - } catch (IOException e) { - log.info("Exception caught while waiting for the HBase server to start", e); - } - after = System.currentTimeMillis(); - } while (timeoutMS > after - before); - after = System.currentTimeMillis(); - log.warn("HBase server did not start in {} ms", after - before); - } - - /** - * Delete HBase data under the current working directory. - */ - private synchronized static void deleteData() { - try { - // please keep in sync with HBASE_CONFIG_DIR/hbase-site.xml, reading HBase XML config is huge pain. - File hbaseRoot = new File("./target/hbase-root"); - File zookeeperDataDir = new File("./target/zk-data"); - - if (hbaseRoot.exists()) { - log.info("Deleting {}", hbaseRoot); - FileUtils.deleteDirectory(hbaseRoot); - } - - if (zookeeperDataDir.exists()) { - log.info("Deleting {}", zookeeperDataDir); - FileUtils.deleteDirectory(zookeeperDataDir); - } - - } catch (IOException e) { - throw new RuntimeException("Failed to delete old HBase test data directories", e); - } - } - - /** - * Register a shutdown hook with the JVM that attempts to kill the external - * HBase daemon - * - * @param stat - * the HBase daemon to kill - */ - private static void registerKillerHook(final HBaseStatus stat) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdownHBase(stat))); - } - - /** - * Runs the {@code hbase-daemon.sh stop master} script corresponding to the - * HBase version described by the parameter. - * - * @param stat - * the running HBase daemon to stop - */ - private synchronized static void shutdownHBase(HBaseStatus stat) { - - log.info("Shutting down HBase..."); - - // First try graceful shutdown through the script... - DaemonRunner.runCommand(stat.getScriptDir() + "/hbase-daemon.sh", "--config", stat.getConfDir(), "stop", "master"); - - log.info("Shutdown HBase"); - - if (!stat.getFile().delete()) { - log.warn("Unable to delete stat file {}", stat.getFile().getAbsolutePath()); - } - - log.info("Deleted {}", stat.getFile()); - - HBASE = null; - } - - /** - * Create a snapshot for a table. - * @param snapshotName - * @param table - * @throws BackendException - */ - public synchronized static void createSnapshot(String snapshotName, String table) - throws BackendException { - try (ConnectionMask hc = compat.createConnection(HBaseConfiguration.create()); - AdminMask admin = hc.getAdmin()) { - admin.snapshot(snapshotName, table); - } catch (Exception e) { - log.warn("Create HBase snapshot failed", e); - throw new TemporaryBackendException("Create HBase snapshot failed", e); - } - } - - /** - * Delete a snapshot. - * @param snapshotName - * @throws IOException - */ - public synchronized static void deleteSnapshot(String snapshotName) throws IOException { - try (ConnectionMask hc = compat.createConnection(HBaseConfiguration.create()); - AdminMask admin = hc.getAdmin()) { - admin.deleteSnapshot(snapshotName); - } - } - - /** - * Return the hbase root dir - */ - public static String getHBaseRootdir() { - return getDirForHBaseVersion(HBASE_TARGET_VERSION, "target/hbase-root"); - } -} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java b/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java deleted file mode 100644 index 55b56b8bfa..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph.blueprints; - -import org.janusgraph.HBaseStorageSetup; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; - -import java.io.IOException; - -/** - * @author Matthias Broecheler (me@matthiasb.com) - */ -public class HBaseGraphProvider extends AbstractJanusGraphProvider { - - @Override - public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class test, String testMethodName) { - try { - HBaseStorageSetup.startHBase(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return HBaseStorageSetup.getHBaseConfiguration(graphName); - } -} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/pom.xml b/janusgraph-hbase-parent/janusgraph-hbase-server/pom.xml deleted file mode 100644 index b953c005c3..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/pom.xml +++ /dev/null @@ -1,143 +0,0 @@ - - 4.0.0 - - org.janusgraph - janusgraph-hbase-parent - 0.6.0-SNAPSHOT - ../pom.xml - - janusgraph-hbase-server - JanusGraph-HBase: HBase Server Used by Maven Testing - https://janusgraph.org - - - ${basedir}/../.. - - - - - org.apache.hbase - hbase-server - ${hbase.server.version} - true - - - org.apache.commons - commons-compress - - - - - - - - - - com.google.guava - guava - 14.0 - - - org.jruby.jcodings - jcodings - 1.0.55 - - - org.glassfish - javax.el - 3.0.0 - - - - - - src/main/java - ../common/test/java - - - ${project.basedir}/../common/test/resources - true - - - - - maven-resources-plugin - - - filter-hbase-conf - - - - - maven-dependency-plugin - - - generate-cached-classpath - - - unpack-common-test-classes - - - - - maven-assembly-plugin - - - copy-hbase-bin - - - - - maven-clean-plugin - - - clean-hbase-files - - - - - maven-surefire-plugin - - - - default-test - none - - - tinkerpop-test - none - - - - - maven-deploy-plugin - - - default-deploy - none - - - - - - - - - hbase1 - - ${hbase1.version} - - - - hbase2 - - - !hbase.profile - - - - ${hbase2.version} - - - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/graceful_stop.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/graceful_stop.sh deleted file mode 100755 index 98b27e31b5..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/graceful_stop.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Move regions off a server then stop it. Optionally restart and reload. -# Turn off the balancer before running this script. -function usage { - echo "Usage: graceful_stop.sh [--config ] [-d] [-e] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " d|debug Print helpful debug information" - echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." - echo " hostname Hostname of server we are to stop" - echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" - exit 1 -} - -if [ $# -lt 1 ]; then - usage -fi - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh -# Get arguments -restart= -reload= -debug= -thrift= -rest= -maxthreads=1 -failfast= -while [ $# -gt 0 ] -do - case "$1" in - --thrift) thrift=true; shift;; - --rest) rest=true; shift;; - --restart) restart=true; shift;; - --reload) reload=true; shift;; - --failfast | -e) failfast=true; shift;; - --debug | -d) debug="--debug"; shift;; - --maxthreads) shift; maxthreads=$1; shift;; - --) shift; break;; - -*) usage ;; - *) break;; # terminate while loop - esac -done - -# "$@" contains the rest. Must be at least the hostname left. -if [ $# -lt 1 ]; then - usage -fi - -# Emit a log line w/ iso8901 date prefixed -log() { - echo `date +%Y-%m-%dT%H:%M:%S` $1 -} - -# See if we should set fail fast before we do anything. -if [ "$failfast" != "" ]; then - log "Set failfast, will exit immediately if any command exits with non-zero status" - set -e -fi - -hostname=$1 -filename="/tmp/$hostname" - -local=false -localhostname=`/bin/hostname` - -if [ "$localhostname" == "$hostname" ]; then - local=true -fi - -log "Disabling load balancer" -HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` -log "Previous balancer state was $HBASE_BALANCER_STATE" - -log "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads unload $hostname -log "Unloaded $hostname region(s)" - -# Stop the server(s). Have to put hostname into its own little file for hbase-daemons.sh -hosts="/tmp/$(basename $0).$$.tmp" -echo $hostname >> $hosts -if [ "$thrift" != "" ]; then - log "Stopping thrift" - if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop thrift - else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift - fi -fi -if [ "$rest" != "" ]; then - log "Stopping rest" - if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop rest - else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest - fi -fi -log "Stopping regionserver" -if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop regionserver -else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver -fi -if [ "$restart" != "" ]; then - log "Restarting regionserver" - if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start regionserver - else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver - fi - if [ "$thrift" != "" ]; then - log "Restarting thrift" - # -b 0.0.0.0 says listen on all interfaces rather than just default. - if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start thrift -b 0.0.0.0 - else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 - fi - fi - if [ "$rest" != "" ]; then - log "Restarting rest" - if [ "$local" ]; then - "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start rest - else - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest - fi - fi - if [ "$reload" != "" ]; then - log "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads load $hostname - log "Reloaded $hostname region(s)" - fi -fi - -# Restore balancer state -if [ $HBASE_BALANCER_STATE != "false" ]; then - log "Restoring balancer state to " $HBASE_BALANCER_STATE - echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null -fi - -# Cleanup tmp files. -trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase deleted file mode 100755 index 7fe17521a8..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase +++ /dev/null @@ -1,421 +0,0 @@ -#! /usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# The hbase command script. Based on the hadoop command script putting -# in hbase classes, libs and configurations ahead of hadoop's. -# -# TODO: Narrow the amount of duplicated code. -# -# Environment Variables: -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HBASE_CLASSPATH Extra Java CLASSPATH entries. -# -# HBASE_CLASSPATH_PREFIX Extra Java CLASSPATH entries that should be -# prefixed to the system classpath. -# -# HBASE_HEAPSIZE The maximum amount of heap to use. -# Default is unset and uses the JVMs default setting -# (usually 1/4th of the available memory). -# -# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding -# native libraries. -# -# HBASE_OPTS Extra Java runtime options. -# -# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -# -# HBASE_ROOT_LOGGER The root appender. Default is INFO,console -# -# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. -# Defaults to the jar packaged with HBase. -# -# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -# Empty by default. -# -# HBASE_SHELL_OPTS Extra options passed to the hbase shell. -# Empty by default. -# -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -# Detect if we are in hbase sources dir -in_dev_env=false -if [ -d "${HBASE_HOME}/target" ]; then - in_dev_env=true -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hbase [] []" - echo "Options:" - echo " --config DIR Configuration direction to use. Default: ./conf" - echo " --hosts HOSTS Override the list in 'regionservers' file" - echo "" - echo "Commands:" - echo "Some commands take arguments. Pass no args or -h for usage." - echo " shell Run the HBase shell" - echo " hbck Run the hbase 'fsck' tool" - echo " snapshot Create a new snapshot of a table" - echo " wal Write-ahead-log analyzer" - echo " hfile Store file analyzer" - echo " zkcli Run the ZooKeeper shell" - echo " upgrade Upgrade hbase" - echo " master Run an HBase HMaster node" - echo " regionserver Run an HBase HRegionServer node" - echo " zookeeper Run a Zookeeper server" - echo " rest Run an HBase REST server" - echo " thrift Run the HBase Thrift server" - echo " thrift2 Run the HBase Thrift2 server" - echo " clean Run the HBase clean up script" - echo " classpath Dump hbase CLASSPATH" - echo " mapredcp Dump CLASSPATH entries required by mapreduce" - echo " pe Run PerformanceEvaluation" - echo " ltt Run LoadTestTool" - echo " version Print the version" - echo " CLASSNAME Run the class named CLASSNAME" - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -JAVA=$JAVA_HOME/bin/java - -# override default settings for this command, if applicable -if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then - . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" -fi - -add_size_suffix() { - # add an 'm' suffix if the argument is missing one, otherwise use whats there - local val="$1" - local lastchar=${val: -1} - if [[ "mMgG" == *$lastchar* ]]; then - echo $val - else - echo ${val}m - fi -} - -if [[ -n "$HBASE_HEAPSIZE" ]]; then - JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_HEAPSIZE)" -fi - -if [[ -n "$HBASE_OFFHEAPSIZE" ]]; then - JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)" -fi - -# so that filenames w/ spaces are handled correctly in loops below -ORIG_IFS=$IFS -IFS= - -# CLASSPATH initially contains $HBASE_CONF_DIR -CLASSPATH="${HBASE_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -add_to_cp_if_exists() { - if [ -d "$@" ]; then - CLASSPATH=${CLASSPATH}:"$@" - fi -} - -# For releases, add hbase & webapps to CLASSPATH -# Webapps must come first else it messes up Jetty -if [ -d "$HBASE_HOME/hbase-webapps" ]; then - add_to_cp_if_exists "${HBASE_HOME}" -fi -#add if we are in a dev environment -if [ -d "$HBASE_HOME/hbase-server/target/hbase-webapps" ]; then - add_to_cp_if_exists "${HBASE_HOME}/hbase-server/target" -fi - -add_maven_deps_to_classpath() { - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - echo "As this is a development environment, we need ${f} to be generated from maven (command: mvn install -DskipTests)" - exit 1 - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - - -add_maven_deps_to_classpath() { - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ]; then - # ------------------- - # JanusGraph build support - # ------------------- - # Check for Maven command line options file. - # If janusgraph-hbase has already been built, then this file will contain options - # to target the version of Hadoop used during compilation. This is - # necessary when the user compiles janusgraph-hbase with a non-default Hadoop - # version (e.g. 2.2.0). If we omitted this check, then we'd try to start - # HBase with the default Hadoop version (currently 1.0.4), and it would break. - MVN_OPTS= - optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" - unset IFS - if [ -f "${optf}" ]; then - MVN_OPTS=`cat "${optf}"` - fi - mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - - -#Add the development env class path stuff -if $in_dev_env; then - add_maven_deps_to_classpath -fi - -#add the hbase jars for each module -for f in $HBASE_HOME/hbase-jars/hbase*.jar; do - if [[ $f = *sources.jar ]] - then - : # Skip sources.jar - elif [ -f $f ] - then - CLASSPATH=${CLASSPATH}:$f; - fi -done - -# Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# default log directory & file -if [ "$HBASE_LOG_DIR" = "" ]; then - HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -if [ "$HBASE_LOGFILE" = "" ]; then - HBASE_LOGFILE='hbase.log' -fi - -function append_path() { - if [ -z "$1" ]; then - echo $2 - else - echo $1:$2 - fi -} - -JAVA_PLATFORM="" - -# if HBASE_LIBRARY_PATH is defined lets use it as first or second option -if [ "$HBASE_LIBRARY_PATH" != "" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") -fi - -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -# Allow this functionality to be disabled -if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then - HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) - if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) - fi -fi - -# Add user-specified CLASSPATH last -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} -fi - -# Add user-specified CLASSPATH prefix first -if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then - CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH} -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - -if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then - if [ -z $JAVA_PLATFORM ]; then - JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - fi - if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) - fi - - if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) - fi -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -# restore ordinary behaviour -unset IFS - -#Set the right GC options based on the what we are running -declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") -for cmd in ${server_cmds[@]}; do - if [[ $cmd == $COMMAND ]]; then - server=true - break - fi -done - -if [[ $server ]]; then - HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" -else - HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" -fi - -# figure out which class to run -if [ "$COMMAND" = "shell" ] ; then - # eg export JRUBY_HOME=/usr/local/share/jruby - if [ "$JRUBY_HOME" != "" ] ; then - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi - # find the hbase ruby sources - HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" -elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -# TODO remove old 'hlog' version -elif [ "$COMMAND" = "hlog" -o "$COMMAND" = "wal" ] ; then - CLASS='org.apache.hadoop.hbase.wal.WALPrettyPrinter' -elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter' -elif [ "$COMMAND" = "zkcli" ] ; then - CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer" -elif [ "$COMMAND" = "upgrade" ] ; then - CLASS="org.apache.hadoop.hbase.migration.UpgradeTo96" -elif [ "$COMMAND" = "snapshot" ] ; then - CLASS="org.apache.hadoop.hbase.snapshot.CreateSnapshot" -elif [ "$COMMAND" = "master" ] ; then - CLASS='org.apache.hadoop.hbase.master.HMaster' - if [ "$1" != "stop" ] && [ "$1" != "clear" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" - fi -elif [ "$COMMAND" = "regionserver" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" - fi -elif [ "$COMMAND" = "thrift" ] ; then - CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "thrift2" ] ; then - CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "rest" ] ; then - CLASS='org.apache.hadoop.hbase.rest.RESTServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" - fi -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" - fi -elif [ "$COMMAND" = "clean" ] ; then - case $1 in - --cleanZk|--cleanHdfs|--cleanAll) - matches="yes" ;; - *) ;; - esac - if [ $# -ne 1 -o "$matches" = "" ]; then - echo "Usage: hbase clean (--cleanZk|--cleanHdfs|--cleanAll)" - echo "Options: " - echo " --cleanZk cleans hbase related data from zookeeper." - echo " --cleanHdfs cleans hbase related data from hdfs." - echo " --cleanAll cleans hbase related data from both zookeeper and hdfs." - exit 1; - fi - "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ - exit $? -elif [ "$COMMAND" = "mapredcp" ] ; then - CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' -elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit 0 -elif [ "$COMMAND" = "pe" ] ; then - CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' - HBASE_OPTS="$HBASE_OPTS $HBASE_PE_OPTS" -elif [ "$COMMAND" = "ltt" ] ; then - CLASS='org.apache.hadoop.hbase.util.LoadTestTool' - HBASE_OPTS="$HBASE_OPTS $HBASE_LTT_OPTS" -elif [ "$COMMAND" = "version" ] ; then - CLASS='org.apache.hadoop.hbase.util.VersionInfo' -else - CLASS=$COMMAND -fi - -# Have JVM dump heap if we run out of memory. Files will be 'launch directory' -# and are named like the following: java_pid21612.hprof. Apparently it doesn't -# 'cost' to have this flag enabled. Its a 1.6 flag only. See: -# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" -HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" -HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" -HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" -fi - -# Enable security logging on the master and regionserver only -if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,RFAS}" -else - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" -fi - -HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" -# Exec unless HBASE_NOEXEC is set. -export CLASSPATH -if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" -else - exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" -fi diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-cleanup.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-cleanup.sh deleted file mode 100755 index 596e366a30..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-cleanup.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# cleans hbase related data from zookeeper and hdfs if no hbase process is alive. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# - -usage="Usage: hbase-cleanup.sh (--cleanZk|--cleanHdfs|--cleanAll|--cleanAcls)" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -case $1 in - --cleanZk|--cleanHdfs|--cleanAll|--cleanAcls) - matches="yes" ;; - *) ;; -esac -if [ $# -ne 1 -o "$matches" = "" ]; then - echo $usage - exit 1; -fi - -format_option=$1; - -zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` -if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - -hrootdir=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.rootdir` -if [ "$hrootdir" == "null" ]; then hrootdir="file:///tmp/hbase-${USER}/hbase"; fi - -check_for_znodes() { - command=$1; - case $command in - regionservers) - zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zchild" == "null" ]; then zchild="rs"; fi - ;; - backupmasters) - zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.backup.masters` - if [ "$zchild" == "null" ]; then zchild="backup-masters"; fi - ;; - esac - znodes=`"$bin"/hbase zkcli ls $zparent/$zchild 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - if [ "$znodes" != "" ]; then - echo -n "ZNode(s) [${znodes}] of $command are not expired. Exiting without cleaning hbase data." - echo #force a newline - exit 1; - else - echo -n "All ZNode(s) of $command are expired." - fi - echo #force a newline -} - -execute_zk_command() { - command=$1; - "$bin"/hbase zkcli $command 2>&1 -} - -execute_hdfs_command() { - command=$1; - "$bin"/hbase org.apache.hadoop.fs.FsShell $command 2>&1 -} - -execute_clean_acls() { - command=$1; - "$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZkAclReset $command 2>&1 -} - -clean_up() { - case $1 in - --cleanZk) - execute_zk_command "rmr ${zparent}"; - ;; - --cleanHdfs) - execute_hdfs_command "-rmr ${hrootdir}" - ;; - --cleanAll) - execute_zk_command "rmr ${zparent}"; - execute_hdfs_command "-rmr ${hrootdir}" - ;; - --cleanAcls) - execute_clean_acls; - ;; - *) - ;; - esac -} - -check_znode_exists() { - command=$1 - "$bin"/hbase zkcli stat $command 2>&1 | grep "Node does not exist\|Connection refused" -} - -check_znode_exists $zparent -if [ $? -ne 0 ]; then - # make sure the online region server(s) znode(s) have been deleted before continuing - check_for_znodes regionservers - # make sure the backup master(s) znode(s) has been deleted before continuing - check_for_znodes backupmasters - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - check_znode_exists $zmaster - if [ $? -ne 0 ]; then - echo -n "Master ZNode is not expired. Exiting without cleaning hbase data." - echo #force a new line - exit 1 - else - echo "Active Master ZNode also expired." - fi - echo #force a newline -else - echo "HBase parent znode ${zparent} does not exist." -fi - -# cleans zookeeper and/or hdfs data. -clean_up $format_option diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-common.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-common.sh deleted file mode 100755 index 0a474f7304..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-common.sh +++ /dev/null @@ -1,41 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## - -#Shared function to wait for a process end. Take the pid and the command name as parameters -waitForProcessEnd() { - pidKilled=$1 - commandName=$2 - processedAt=`date +%s` - while kill -0 $pidKilled > /dev/null 2>&1; - do - echo -n "." - sleep 1; - # if process persists more than $HBASE_STOP_TIMEOUT (default 1200 sec) no mercy - if [ $(( `date +%s` - $processedAt )) -gt ${HBASE_STOP_TIMEOUT:-1200} ]; then - break; - fi - done - # process still there : kill -9 - if kill -0 $pidKilled > /dev/null 2>&1; then - echo -n force stopping $commandName with kill -9 $pidKilled - $JAVA_HOME/bin/jstack -l $pidKilled > "$logout" 2>&1 - kill -9 $pidKilled > /dev/null 2>&1 - fi - # Add a CR after we're done w/ dots. - echo -} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.cmd b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.cmd deleted file mode 100755 index 5c1f1860f5..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.cmd +++ /dev/null @@ -1,78 +0,0 @@ -@rem/* -@rem * Licensed to the Apache Software Foundation (ASF) under one -@rem * or more contributor license agreements. See the NOTICE file -@rem * distributed with this work for additional information -@rem * regarding copyright ownership. The ASF licenses this file -@rem * to you under the Apache License, Version 2.0 (the -@rem * "License"); you may not use this file except in compliance -@rem * with the License. You may obtain a copy of the License at -@rem * -@rem * http://www.apache.org/licenses/LICENSE-2.0 -@rem * -@rem * Unless required by applicable law or agreed to in writing, software -@rem * distributed under the License is distributed on an "AS IS" BASIS, -@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem * See the License for the specific language governing permissions and -@rem * limitations under the License. -@rem */ - -@rem included in all the hbase scripts with source command -@rem should not be executable directly -@rem also should not be passed any arguments, since we need original $* -@rem Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -@rem Make sure java environment is set -@rem - -if "%HBASE_BIN_PATH:~-1%" == "\" ( - set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% -) - -if "%1" == "--config" ( - set HBASE_CONF_DIR=%2 - shift - shift -) - -@rem the root of the hbase installation -if not defined HBASE_HOME ( - set HBASE_HOME=%HBASE_BIN_PATH%\.. -) - -@rem Allow alternate hbase conf dir location. -if not defined HBASE_CONF_DIR ( - set HBASE_CONF_DIR=%HBASE_HOME%\conf -) - -@rem List of hbase regions servers. -if not defined HBASE_REGIONSERVERS ( - set HBASE_REGIONSERVERS=%HBASE_CONF_DIR%\regionservers -) - -@rem List of hbase secondary masters. -if not defined HBASE_BACKUP_MASTERS ( - set HBASE_BACKUP_MASTERS=%HBASE_CONF_DIR%\backup-masters -) - -@rem Source the hbase-env.sh. Will have JAVA_HOME defined. -if exist "%HBASE_CONF_DIR%\hbase-env.cmd" ( - call "%HBASE_CONF_DIR%\hbase-env.cmd" -) - -if not defined JAVA_HOME ( - echo Warning: JAVA_HOME environment variable is not set. Defaulting to c:\apps\java - set JAVA_HOME=c:\apps\java -) - -if not exist "%JAVA_HOME%\bin\java.exe" ( - echo Error: JAVA_HOME is incorrectly set or could not find java at the location %JAVA_HOME%\bin\ - exit /B 2 -) - -set JAVA="%JAVA_HOME%\bin\java" - -for %%i in (%0) do ( - if not defined HBASE_BIN_PATH ( - set HBASE_BIN_PATH=%%~dpi - ) -) \ No newline at end of file diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.sh deleted file mode 100755 index 58ce61b6c1..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-config.sh +++ /dev/null @@ -1,136 +0,0 @@ -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# included in all the hbase scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* -# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -# resolve links - "${BASH_SOURCE-$0}" may be a softlink - -this="${BASH_SOURCE-$0}" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin">/dev/null; pwd` -this="$bin/$script" - -# the root of the hbase installation -if [ -z "$HBASE_HOME" ]; then - export HBASE_HOME=`dirname "$this"`/.. -fi - -#check to see if the conf dir or hbase home are given as an optional arguments -while [ $# -gt 1 ] -do - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HBASE_CONF_DIR=$confdir - elif [ "--hosts" = "$1" ] - then - shift - hosts=$1 - shift - HBASE_REGIONSERVERS=$hosts - else - # Presume we are at end of options and break - break - fi -done - -# Allow alternate hbase conf dir location. -HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" -# List of hbase regions servers. -HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" -# List of hbase secondary masters. -HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" -# Thrift JMX opts -if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_THRIFT_JMX_OPTS" ]; then - HBASE_THRIFT_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10103" -fi -# Thrift opts -if [ -z "$HBASE_THRIFT_OPTS" ]; then - export HBASE_THRIFT_OPTS="$HBASE_THRIFT_JMX_OPTS" -fi - -# REST JMX opts -if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_REST_JMX_OPTS" ]; then - HBASE_REST_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10105" -fi -# REST opts -if [ -z "$HBASE_REST_OPTS" ]; then - export HBASE_REST_OPTS="$HBASE_REST_JMX_OPTS" -fi - -# Source the hbase-env.sh. Will have JAVA_HOME defined. -# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. -if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then - . "${HBASE_CONF_DIR}/hbase-env.sh" - export HBASE_ENV_INIT="true" -fi - -# Set default value for regionserver uid if not present -if [ -z "$HBASE_REGIONSERVER_UID" ]; then - HBASE_REGIONSERVER_UID="hbase" -fi - -# Verify if hbase has the mlock agent -if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then - MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so" - if [ ! -f "$MLOCK_AGENT" ]; then - cat 1>&2 <&2 < https://www.oracle.com/technetwork/java/javase/downloads | -| | -| HBase requires Java 1.7 or later. | -+======================================================================+ -EOF - exit 1 -fi diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemon.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemon.sh deleted file mode 100755 index 6797e1ca69..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemon.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Runs a Hadoop hbase command as a daemon. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_LOG_DIR Where log files are stored. PWD by default. -# HBASE_PID_DIR The pid files are stored. /tmp by default. -# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default -# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. -# HBASE_STOP_TIMEOUT Time, in seconds, after which we kill -9 the server if it has not stopped. -# Default 1200 seconds. -# -# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh - -usage="Usage: hbase-daemon.sh [--config ]\ - (start|stop|restart|autorestart) \ - " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh -. "$bin"/hbase-common.sh - -# get arguments -startStop=$1 -shift - -command=$1 -shift - -hbase_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" - num=$prev - done - mv -f "$log" "$log.$num"; - fi -} - -cleanZNode() { - if [ -f $HBASE_ZNODE_FILE ]; then - if [ "$command" = "master" ]; then - $bin/hbase master clear > /dev/null 2>&1 - else - #call ZK to delete the node - ZNODE=`cat $HBASE_ZNODE_FILE` - $bin/hbase zkcli delete $ZNODE > /dev/null 2>&1 - fi - rm $HBASE_ZNODE_FILE - fi -} - -check_before_start(){ - #check if the process is not running - mkdir -p "$HBASE_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi -} - -wait_until_done () -{ - p=$1 - cnt=${HBASE_SLAVE_TIMEOUT:-300} - origcnt=$cnt - while kill -0 $p > /dev/null 2>&1; do - if [ $cnt -gt 1 ]; then - cnt=`expr $cnt - 1` - sleep 1 - else - echo "Process did not complete after $origcnt seconds, killing." - kill -9 $p - exit 1 - fi - done - return 0 -} - -# get log directory -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_PID_DIR" = "" ]; then - HBASE_PID_DIR=/tmp -fi - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -# Some variables -# Work out java location so can print version into log. -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log - -if [ -z "${HBASE_ROOT_LOGGER}" ]; then -export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"} -fi - -if [ -z "${HBASE_SECURITY_LOGGER}" ]; then -export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"} -fi - -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid -export HBASE_ZNODE_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.znode -export HBASE_START_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.autorestart - -if [ -n "$SERVER_GC_OPTS" ]; then - export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi -if [ -n "$CLIENT_GC_OPTS" ]; then - export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi - -# Set default scheduling priority -if [ "$HBASE_NICENESS" = "" ]; then - export HBASE_NICENESS=0 -fi - -thiscmd="$bin/$(basename ${BASH_SOURCE-$0})" -args=$@ - -case $startStop in - -(start) - check_before_start - hbase_rotate_log $logout - hbase_rotate_log $loggc - echo starting $command, logging to $logout - nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args < /dev/null > ${logout} 2>&1 & - sleep 1; head "${logout}" - ;; - -(autorestart) - check_before_start - hbase_rotate_log $logout - hbase_rotate_log $loggc - nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_autorestart $command $args < /dev/null > ${logout} 2>&1 & - ;; - -(internal_start) - # Add to the command log file vital stats on our environment. - echo "`date` Starting $command on `hostname`" >> $loglog - echo "`ulimit -a`" >> $loglog 2>&1 - nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - $command "$@" start >> "$logout" 2>&1 & - echo $! > $pid - wait - cleanZNode - ;; - -(internal_autorestart) - touch "$HBASE_START_FILE" - #keep starting the command until asked to stop. Reloop on software crash - while true - do - lastLaunchDate=`date +%s` - $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args - - #if the file does not exist it means that it was not stopped properly by the stop command - if [ ! -f "$HBASE_START_FILE" ]; then - exit 1 - fi - - #if the cluster is being stopped then do not restart it again. - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - zkrunning=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.state` - if [ "$zkrunning" == "null" ]; then zkrunning="running"; fi - zkFullRunning=$zparent/$zkrunning - $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep "Node does not exist" 1>/dev/null 2>&1 - #grep returns 0 if it found something, 1 otherwise - if [ $? -eq 0 ]; then - exit 1 - fi - - #If ZooKeeper cannot be found, then do not restart - $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep Exception | grep ConnectionLoss 1>/dev/null 2>&1 - if [ $? -eq 0 ]; then - exit 1 - fi - - #if it was launched less than 5 minutes ago, then wait for 5 minutes before starting it again. - curDate=`date +%s` - limitDate=`expr $lastLaunchDate + 300` - if [ $limitDate -gt $curDate ]; then - sleep 300 - fi - done - ;; - -(stop) - rm -f "$HBASE_START_FILE" - if [ -f $pid ]; then - pidToKill=`cat $pid` - # kill -0 == see if the PID exists - if kill -0 $pidToKill > /dev/null 2>&1; then - echo -n stopping $command - echo "`date` Terminating $command" >> $loglog - kill $pidToKill > /dev/null 2>&1 - waitForProcessEnd $pidToKill $command - else - retval=$? - echo no $command to stop because kill -0 of pid $pidToKill failed with status $retval - fi - else - echo no $command to stop because no pid file $pid - fi - rm -f $pid - ;; - -(restart) - # stop the command - $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & - wait_until_done $! - # wait a user-specified sleep period - sp=${HBASE_RESTART_SLEEP:-3} - if [ $sp -gt 0 ]; then - sleep $sp - fi - # start the command - $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & - wait_until_done $! - ;; - -(*) - echo $usage - exit 1 - ;; -esac diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemons.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemons.sh deleted file mode 100755 index 21ce635777..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-daemons.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [--hosts regionserversfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. $bin/hbase-config.sh - -remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" -args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" - -command=$2 -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (master-backup) - exec "$bin/master-backup.sh" $args - ;; - (*) - exec "$bin/regionservers.sh" $args - ;; -esac - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-jruby b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-jruby deleted file mode 100755 index 37bce46f6b..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase-jruby +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - - -`dirname $0`/hbase org.jruby.Main $* - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase.cmd b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase.cmd deleted file mode 100755 index da7bcd9dea..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/hbase.cmd +++ /dev/null @@ -1,439 +0,0 @@ -@echo off -@rem/* -@rem * Licensed to the Apache Software Foundation (ASF) under one -@rem * or more contributor license agreements. See the NOTICE file -@rem * distributed with this work for additional information -@rem * regarding copyright ownership. The ASF licenses this file -@rem * to you under the Apache License, Version 2.0 (the -@rem * "License"); you may not use this file except in compliance -@rem * with the License. You may obtain a copy of the License at -@rem * -@rem * http://www.apache.org/licenses/LICENSE-2.0 -@rem * -@rem * Unless required by applicable law or agreed to in writing, software -@rem * distributed under the License is distributed on an "AS IS" BASIS, -@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem * See the License for the specific language governing permissions and -@rem * limitations under the License. -@rem */ -@rem -@rem The hbase command script. Based on the hadoop command script putting -@rem in hbase classes, libs and configurations ahead of hadoop's. -@rem -@rem TODO: Narrow the amount of duplicated code. -@rem -@rem Environment Variables: -@rem -@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -@rem -@rem HBASE_CLASSPATH Extra Java CLASSPATH entries. -@rem -@rem HBASE_HEAPSIZE The maximum amount of heap to use. -@rem Default is unset and uses the JVMs default setting -@rem (usually 1/4th of the available memory). -@rem -@rem HBASE_OPTS Extra Java runtime options. -@rem -@rem HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -@rem -@rem HBASE_ROOT_LOGGER The root appender. Default is INFO,console -@rem -@rem JRUBY_HOME JRuby path: $JRUBY_HOME\lib\jruby.jar should exist. -@rem Defaults to the jar packaged with HBase. -@rem -@rem JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -@rem Empty by default. -@rem HBASE_SHELL_OPTS Extra options passed to the hbase shell. -@rem Empty by default. - - -setlocal enabledelayedexpansion - -for %%i in (%0) do ( - if not defined HBASE_BIN_PATH ( - set HBASE_BIN_PATH=%%~dpi - ) -) - -if "%HBASE_BIN_PATH:~-1%" == "\" ( - set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% -) - -rem This will set HBASE_HOME, etc. -set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd -call "%hbase-config-script%" %* -if "%1" == "--config" ( - shift - shift -) - -rem Detect if we are in hbase sources dir -set in_dev_env=false - -if exist "%HBASE_HOME%\target" set in_dev_env=true - -rem --service is an internal option. used by MSI setup to install HBase as a windows service -if "%1" == "--service" ( - set service_entry=true - shift -) - -set hbase-command=%1 -shift - -@rem if no args specified, show usage -if "%hbase-command%"=="" ( - goto :print_usage - endlocal - goto :eof -) - -set JAVA_HEAP_MAX= -set JAVA_OFFHEAP_MAX= - -rem check envvars which might override default args -if defined HBASE_HEAPSIZE ( - set JAVA_HEAP_MAX=-Xmx%HBASE_HEAPSIZE%m -) - -if defined HBASE_OFFHEAPSIZE ( - set JAVA_OFFHEAP_MAX=-XX:MaxDirectMemory=%HBASE_OFFHEAPSIZE%m -) - -set CLASSPATH=%HBASE_CONF_DIR%;%JAVA_HOME%\lib\tools.jar - -rem Add maven target directory -set cached_classpath_filename=%HBASE_HOME%\target\cached_classpath.txt -if "%in_dev_env%"=="true" ( - - rem adding maven main classes to classpath - for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( - if exist %%i\target\classes set CLASSPATH=!CLASSPATH!;%%i\target\classes - ) - - rem adding maven test classes to classpath - rem For developers, add hbase classes to CLASSPATH - for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( - if exist %%i\target\test-classes set CLASSPATH=!CLASSPATH!;%%i\target\test-classes - ) - - if not exist "%cached_classpath_filename%" ( - echo "As this is a development environment, we need %cached_classpath_filename% to be generated from maven (command: mvn install -DskipTests)" - goto :eof - ) - - for /f "delims=" %%i in ('type "%cached_classpath_filename%"') do set CLASSPATH=%CLASSPATH%;%%i -) - -@rem For releases add hbase webapps to CLASSPATH -@rem Webapps must come first else it messes up Jetty -if exist "%HBASE_HOME%\hbase-webapps" ( - set CLASSPATH=%CLASSPATH%;%HBASE_HOME% -) - -if exist "%HBASE_HOME%\target\hbase-webapps" ( - set CLASSPATH=%CLASSPATH%;%HBASE_HOME%\target -) - -for /F %%f in ('dir /b "%HBASE_HOME%\hbase*.jar" 2^>nul') do ( - if not "%%f:~-11"=="sources.jar" ( - set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\%%f - ) -) - -@rem Add libs to CLASSPATH -if exist "%HBASE_HOME%\lib" ( - set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\lib\* -) - -@rem Add user-specified CLASSPATH last -if defined HBASE_CLASSPATH ( - set CLASSPATH=%CLASSPATH%;%HBASE_CLASSPATH% -) - -@rem Default log directory and file -if not defined HBASE_LOG_DIR ( - set HBASE_LOG_DIR=%HBASE_HOME%\logs -) - -if not defined HBASE_LOGFILE ( - set HBASE_LOGFILE=hbase.log -) - -set JAVA_PLATFORM= - -rem If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -set PATH=%PATH%;"%HADOOP_HOME%\bin" -set HADOOP_IN_PATH=hadoop.cmd - -if exist "%HADOOP_HOME%\bin\%HADOOP_IN_PATH%" ( - set hadoopCpCommand=call %HADOOP_IN_PATH% classpath 2^>nul - for /f "eol= delims=" %%i in ('!hadoopCpCommand!') do set CLASSPATH_FROM_HADOOP=%%i - if defined CLASSPATH_FROM_HADOOP ( - set CLASSPATH=%CLASSPATH%;!CLASSPATH_FROM_HADOOP! - ) - set HADOOP_CLASSPATH=%CLASSPATH% - - set hadoopJLPCommand=call %HADOOP_IN_PATH% org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2^>nul - for /f "eol= delims=" %%i in ('!hadoopJLPCommand!') do set HADOOP_JAVA_LIBRARY_PATH=%%i - if not defined JAVA_LIBRARY_PATH ( - set JAVA_LIBRARY_PATH=!HADOOP_JAVA_LIBRARY_PATH! - ) else ( - set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!HADOOP_JAVA_LIBRARY_PATH! - ) -) - -if exist "%HBASE_HOME%\build\native" ( - set platformCommand=call %JAVA% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName - for /f %%i in ('!platformCommand!') do set JAVA_PLATFORM=%%i - set _PATH_TO_APPEND=%HBASE_HOME%\build\native\!JAVA_PLATFORM!;%HBASE_HOME%\build\native\!JAVA_PLATFORM!\lib - if not defined JAVA_LIBRARY_PATH ( - set JAVA_LIBRARY_PATH=!_PATH_TO_APPEND! - ) else ( - set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!_PATH_TO_APPEND! - ) -) - -rem This loop would set %hbase-command-arguments% -set _hbasearguments= -:MakeCmdArgsLoop - if [%1]==[] goto :EndLoop - - if not defined _hbasearguments ( - set _hbasearguments=%1 - ) else ( - set _hbasearguments=!_hbasearguments! %1 - ) - shift -goto :MakeCmdArgsLoop -:EndLoop - -set hbase-command-arguments=%_hbasearguments% - -@rem figure out which class to run -set corecommands=shell master regionserver thrift thrift2 rest avro hlog wal hbck hfile zookeeper zkcli upgrade mapredcp -for %%i in ( %corecommands% ) do ( - if "%hbase-command%"=="%%i" set corecommand=true -) - -if defined corecommand ( - call :%hbase-command% %hbase-command-arguments% -) else ( - if "%hbase-command%" == "classpath" ( - echo %CLASSPATH% - goto :eof - ) - if "%hbase-command%" == "version" ( - set CLASS=org.apache.hadoop.hbase.util.VersionInfo - ) else ( - set CLASS=%hbase-command% - ) -) - -if not defined HBASE_IDENT_STRING ( - set HBASE_IDENT_STRING=%USERNAME% -) - -@rem Set the right GC options based on the what we are running -set servercommands=master regionserver thrift thrift2 rest avro zookeeper -for %%i in ( %servercommands% ) do ( - if "%hbase-command%"=="%%i" set servercommand=true -) - -if "%servercommand%" == "true" ( - set HBASE_OPTS=%HBASE_OPTS% %SERVER_GC_OPTS% -) else ( - set HBASE_OPTS=%HBASE_OPTS% %CLIENT_GC_OPTS% -) - -@rem If HBase is run as a windows service, configure logging -if defined service_entry ( - set HBASE_LOG_PREFIX=hbase-%hbase-command%-%COMPUTERNAME% - set HBASE_LOGFILE=!HBASE_LOG_PREFIX!.log - if not defined HBASE_ROOT_LOGGER ( - set HBASE_ROOT_LOGGER=INFO,DRFA - ) - set HBASE_SECURITY_LOGGER=INFO,DRFAS - set loggc=!HBASE_LOG_DIR!\!HBASE_LOG_PREFIX!.gc - set loglog=!HBASE_LOG_DIR!\!HBASE_LOGFILE! - - if "%HBASE_USE_GC_LOGFILE%" == "true" ( - set HBASE_OPTS=%HBASE_OPTS% -Xloggc:"!loggc!" - ) -) - - -@rem Have JVM dump heap if we run out of memory. Files will be 'launch directory' -@rem and are named like the following: java_pid21612.hprof. Apparently it does not -@rem 'cost' to have this flag enabled. Its a 1.6 flag only. See: -@rem http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.dir="%HBASE_LOG_DIR%" -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.file="%HBASE_LOGFILE%" -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.home.dir="%HBASE_HOME%" -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.id.str="%HBASE_IDENT_STRING%" -set HBASE_OPTS=%HBASE_OPTS% -XX:OnOutOfMemoryError="taskkill /F /PID %p" - -if not defined HBASE_ROOT_LOGGER ( - set HBASE_ROOT_LOGGER=INFO,console -) -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger="%HBASE_ROOT_LOGGER%" - -if defined JAVA_LIBRARY_PATH ( - set HBASE_OPTS=%HBASE_OPTS% -Djava.library.path="%JAVA_LIBRARY_PATH%" -) - -rem Enable security logging on the master and regionserver only -if not defined HBASE_SECURITY_LOGGER ( - set HBASE_SECURITY_LOGGER=INFO,NullAppender - if "%hbase-command%"=="master" ( - set HBASE_SECURITY_LOGGER=INFO,DRFAS - ) - if "%hbase-command%"=="regionserver" ( - set HBASE_SECURITY_LOGGER=INFO,DRFAS - ) -) -set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%" - -set HEAP_SETTINGS=%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX% -set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments% - -if defined service_entry ( - call :makeServiceXml %java_arguments% -) else ( - call %JAVA% %java_arguments% -) - -endlocal -goto :eof - -:shell - rem eg export JRUBY_HOME=/usr/local/share/jruby - if defined JRUBY_HOME ( - set CLASSPATH=%CLASSPATH%;%JRUBY_HOME%\lib\jruby.jar - set HBASE_OPTS=%HBASE_OPTS% -Djruby.home="%JRUBY_HOME%" -Djruby.lib="%JRUBY_HOME%\lib" - ) - rem find the hbase ruby sources - if exist "%HBASE_HOME%\lib\ruby" ( - set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\lib\ruby" - ) else ( - set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\hbase-shell\src\main\ruby" - ) - set HBASE_OPTS=%HBASE_OPTS% %HBASE_SHELL_OPTS% - - set CLASS=org.jruby.Main -X+O %JRUBY_OPTS% "%HBASE_HOME%\bin\hirb.rb" - goto :eof - -:master - set CLASS=org.apache.hadoop.hbase.master.HMaster - if NOT "%1"=="stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_MASTER_OPTS% - ) - goto :eof - -:regionserver - set CLASS=org.apache.hadoop.hbase.regionserver.HRegionServer - if NOT "%1"=="stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_REGIONSERVER_OPTS% - ) - goto :eof - -:thrift - set CLASS=org.apache.hadoop.hbase.thrift.ThriftServer - if NOT "%1" == "stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% - ) - goto :eof - -:thrift2 - set CLASS=org.apache.hadoop.hbase.thrift2.ThriftServer - if NOT "%1" == "stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% - ) - goto :eof - -:rest - set CLASS=org.apache.hadoop.hbase.rest.RESTServer - if NOT "%1"=="stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_REST_OPTS% - ) - goto :eof - -:avro - set CLASS=org.apache.hadoop.hbase.avro.AvroServer - if NOT "%1"== "stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_AVRO_OPTS% - ) - goto :eof - -:zookeeper - set CLASS=org.apache.hadoop.hbase.zookeeper.HQuorumPeer - if NOT "%1"=="stop" ( - set HBASE_OPTS=%HBASE_OPTS% %HBASE_ZOOKEEPER_OPTS% - ) - goto :eof - -:hbck - set CLASS=org.apache.hadoop.hbase.util.HBaseFsck - goto :eof - -@rem TODO remove older 'hlog' command -:hlog - set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter - goto :eof - -:wal - set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter - goto :eof - -:hfile - set CLASS=org.apache.hadoop.hbase.io.hfile.HFile - goto :eof - -:zkcli - set CLASS=org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer - goto :eof - -:upgrade - set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96 - goto :eof - -:mapredcp - set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool - goto :eof - -:makeServiceXml - set arguments=%* - @echo ^ - @echo ^%hbase-command%^ - @echo ^%hbase-command%^ - @echo ^This service runs Isotope %hbase-command%^ - @echo ^%JAVA%^ - @echo ^%arguments%^ - @echo ^ - goto :eof - -:print_usage - echo Usage: hbase [^] ^ [^] - echo where ^ an option from one of these categories:: - echo Options: - echo --config DIR Configuration direction to use. Default: ./conf - echo. - echo Commands: - echo Some commands take arguments. Pass no args or -h for usage." - echo shell Run the HBase shell - echo hbck Run the hbase 'fsck' tool - echo wal Write-ahead-log analyzer - echo hfile Store file analyzer - echo zkcli Run the ZooKeeper shell - echo upgrade Upgrade hbase - echo master Run an HBase HMaster node - echo regionserver Run an HBase HRegionServer node - echo zookeeper Run a Zookeeper server - echo rest Run an HBase REST server - echo thrift Run the HBase Thrift server - echo thrift2 Run the HBase Thrift2 server - echo classpath Dump hbase CLASSPATH - echo mapredcp Dump CLASSPATH entries required by mapreduce - echo version Print the version - echo CLASSNAME Run the class named CLASSNAME - goto :eof diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/janusgraph_customizations_to_hbase_script.patch b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/janusgraph_customizations_to_hbase_script.patch deleted file mode 100644 index a84e7282bf..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/janusgraph_customizations_to_hbase_script.patch +++ /dev/null @@ -1,50 +0,0 @@ ---- /home/dalaro/hbase-1.0.2/bin/hbase 2015-08-25 18:34:24.000000000 -0400 -+++ hbase 2015-09-02 15:37:46.000000000 -0400 -@@ -168,6 +168,31 @@ - } - - -+add_maven_deps_to_classpath() { -+ f="${HBASE_HOME}/target/cached_classpath.txt" -+ if [ ! -f "${f}" ]; then -+ # ------------------- -+ # JanusGraph build support -+ # ------------------- -+ # Check for Maven command line options file. -+ # If janusgraph-hbase has already been built, then this file will contain options -+ # to target the version of Hadoop used during compilation. This is -+ # necessary when the user compiles janusgraph-hbase with a non-default Hadoop -+ # version (e.g. 2.2.0). If we omitted this check, then we'd try to start -+ # HBase with the default Hadoop version (currently 1.0.4), and it would break. -+ MVN_OPTS= -+ optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" -+ unset IFS -+ if [ -f "${optf}" ]; then -+ MVN_OPTS=`cat "${optf}"` -+ fi -+ mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ IFS= -+ fi -+ CLASSPATH=${CLASSPATH}:`cat "${f}"` -+} -+ -+ - #Add the development env class path stuff - if $in_dev_env; then - add_maven_deps_to_classpath -@@ -286,13 +311,8 @@ - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi -- #find the hbase ruby sources -- if [ -d "$HBASE_HOME/lib/ruby" ]; then -- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/lib/ruby" -- else -- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/hbase-shell/src/main/ruby" -- fi -- HBASE_OPTS="$HBASE_OPTS $HBASE_SHELL_OPTS" -+ # find the hbase ruby sources -+ HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" - elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-master-backup.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-master-backup.sh deleted file mode 100755 index bd265abedf..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-master-backup.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple masters on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 10 masters (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [--config ] [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1" - exit -fi - -. "$bin"/hbase-config.sh - -# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] -export HBASE_MASTER_OPTS=" " - -run_master () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_MASTER_ARGS="\ - -D hbase.master.info.port=`expr 16010 + $DN` \ - -D hbase.regionserver.port=`expr 16020 + $DN` \ - -D hbase.regionserver.info.port=`expr 16030 + $DN` \ - --backup" - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 master $HBASE_MASTER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - if [[ "$i" =~ ^[0-9]+$ ]]; then - run_master $cmd $i - else - echo "Invalid argument" - fi -done diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-regionservers.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-regionservers.sh deleted file mode 100755 index 78b4195811..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/local-regionservers.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple regionservers on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 100 regionservers (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [--config ] [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1 2" - exit -fi - -. "$bin"/hbase-config.sh - -# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] -export HBASE_REGIONSERVER_OPTS=" " - -run_regionserver () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 16200 + $DN` \ - -D hbase.regionserver.info.port=`expr 16300 + $DN`" - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 regionserver $HBASE_REGIONSERVER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - if [[ "$i" =~ ^[0-9]+$ ]]; then - run_regionserver $cmd $i - else - echo "Invalid argument" - fi -done diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/master-backup.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/master-backup.sh deleted file mode 100755 index feca4ab865..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/master-backup.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all backup master hosts. -# -# Environment Variables -# -# HBASE_BACKUP_MASTERS File naming remote hosts. -# Default is ${HBASE_CONF_DIR}/backup-masters -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the master backup file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_BACKUP_MASTERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_BACKUP_MASTERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" - else - export HOSTLIST="${HBASE_BACKUP_MASTERS}" - fi -fi - - -args=${@// /\\ } -args=${args/master-backup/master} - -if [ -f $HOSTLIST ]; then - for hmaster in `cat "$HOSTLIST"`; do - ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ - 2>&1 | sed "s/^/$hmaster: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/regionservers.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/regionservers.sh deleted file mode 100755 index af26ba95d7..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/regionservers.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: regionservers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the regionservers file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_REGIONSERVERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_REGIONSERVERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/regionservers" - else - export HOSTLIST="${HBASE_REGIONSERVERS}" - fi -fi - -regionservers=`cat "$HOSTLIST"` -if [ "$regionservers" = "localhost" ]; then - "$bin"/local-regionservers.sh start 1 -else - for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/rolling-restart.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/rolling-restart.sh deleted file mode 100755 index b3ac67f050..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/rolling-restart.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage_str="Usage: `basename $0` [--config ] [--rs-only] [--master-only] [--graceful] [--maxthreads xx]" - -function usage() { - echo "${usage_str}" -} - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -RR_RS=1 -RR_MASTER=1 -RR_GRACEFUL=0 -RR_MAXTHREADS=1 - -while [ $# -gt 0 ]; do - case "$1" in - --rs-only|-r) - RR_RS=1 - RR_MASTER=0 - RR_GRACEFUL=0 - shift - ;; - --master-only) - RR_RS=0 - RR_MASTER=1 - RR_GRACEFUL=0 - shift - ;; - --graceful) - RR_RS=0 - RR_MASTER=0 - RR_GRACEFUL=1 - shift - ;; - --maxthreads) - shift - RR_MAXTHREADS=$1 - shift - ;; - --help|-h) - usage - exit 0 - ;; - *) - echo Bad argument: $1 - usage - exit 1 - ;; - esac -done - -# quick function to get a value from the HBase config file -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`HBASE_CONF_DIR=${HBASE_CONF_DIR} $bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'false' ]; then - if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then - echo Cant do selective rolling restart if not running distributed - exit 1 - fi - "$bin"/hbase-daemon.sh restart master -else - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - - if [ $RR_MASTER -eq 1 ]; then - # stop all masters before re-start to avoid races for master znode - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - echo -n "Waiting for Master ZNode ${zmaster} to expire" - while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do - echo -n "." - sleep 1 - done - echo #force a newline - - # all masters are down, now restart - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup - - echo "Wait a minute for master to come up join cluster" - sleep 60 - - # Master joing cluster will start in cleaning out regions in transition. - # Wait until the master has cleaned out regions in transition before - # giving it a bunch of work to do; master is vulnerable during startup - zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` - if [ "$zunassigned" == "null" ]; then zunassigned="region-in-transition"; fi - zunassigned="$zparent/$zunassigned" - echo -n "Waiting for ${zunassigned} to empty" - while true ; do - unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` - if test 0 -eq ${unassigned} - then - break - else - echo -n " ${unassigned}" - fi - sleep 1 - done - fi - - if [ $RR_RS -eq 1 ]; then - # unlike the masters, roll all regionservers one-at-a-time - export HBASE_SLAVE_PARALLEL=false - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" restart regionserver - fi - - if [ $RR_GRACEFUL -eq 1 ]; then - # gracefully restart all online regionservers - zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zkrs" == "null" ]; then zkrs="rs"; fi - zkrs="$zparent/$zkrs" - online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - for rs in $online_regionservers - do - rs_parts=(${rs//,/ }) - hostname=${rs_parts[0]} - echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug --maxthreads "${RR_MAXTHREADS}" "$hostname" - sleep 1 - done - fi -fi diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.cmd b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.cmd deleted file mode 100755 index 676a11e6e2..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.cmd +++ /dev/null @@ -1,61 +0,0 @@ -@rem/** -@rem * Licensed to the Apache Software Foundation (ASF) under one -@rem * or more contributor license agreements. See the NOTICE file -@rem * distributed with this work for additional information -@rem * regarding copyright ownership. The ASF licenses this file -@rem * to you under the Apache License, Version 2.0 (the -@rem * "License"); you may not use this file except in compliance -@rem * with the License. You may obtain a copy of the License at -@rem * -@rem * http://www.apache.org/licenses/LICENSE-2.0 -@rem * -@rem * Unless required by applicable law or agreed to in writing, software -@rem * distributed under the License is distributed on an "AS IS" BASIS, -@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem * See the License for the specific language governing permissions and -@rem * limitations under the License. -@rem */ - -@rem Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -@rem Start hadoop hbase daemons. -@rem Run this on master node. -@echo off -set usage="Usage: start-hbase.cmd" - -setlocal - -for %%i in (%0) do ( - if not defined HBASE_BIN_PATH ( - set HBASE_BIN_PATH=%%~dpi - ) -) - -if "%HBASE_BIN_PATH:~-1%" == "\" ( - set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% -) - -set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd -call %hbase-config-script% - -set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed -for /f %%i in ('%distModeCommand%') do set distMode=%%i - -if "%distMode%"=="false" ( - start "HBase Distribution" %HBASE_BIN_PATH%\hbase.cmd master start -) else ( - if "%distMode%"=="true" ( - @echo This is not implemented yet. Stay tuned. - @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "${HBASE_CONF_DIR}" start zookeeper - @rem call %HBASE_BIN_PATH%\hbase-daemon.cmd --config "${HBASE_CONF_DIR}" start master - - @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_REGIONSERVERS%" start regionserver - @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_BACKUP_MASTERS%" start master-backup - ) else ( - echo ERROR: Could not determine the startup mode. - ) -) - -@rem -------------- End of main script -------------- -endlocal -goto :eof \ No newline at end of file diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.sh deleted file mode 100755 index aeb9469fb7..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/start-hbase.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -# Start hadoop hbase daemons. -# Run this on master node. -usage="Usage: start-hbase.sh" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - - -if [ "$1" = "autorestart" ] -then - commandToRun="autorestart" -else - commandToRun="start" -fi - -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` - - -if [ "$distMode" == 'false' ] -then - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master $@ -else - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" $commandToRun zookeeper - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" $commandToRun regionserver - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" $commandToRun master-backup -fi diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.cmd b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.cmd deleted file mode 100755 index 97180554b5..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.cmd +++ /dev/null @@ -1,54 +0,0 @@ -@echo off -@rem/** -@rem * Licensed to the Apache Software Foundation (ASF) under one -@rem * or more contributor license agreements. See the NOTICE file -@rem * distributed with this work for additional information -@rem * regarding copyright ownership. The ASF licenses this file -@rem * to you under the Apache License, Version 2.0 (the -@rem * "License"); you may not use this file except in compliance -@rem * with the License. You may obtain a copy of the License at -@rem * -@rem * http://www.apache.org/licenses/LICENSE-2.0 -@rem * -@rem * Unless required by applicable law or agreed to in writing, software -@rem * distributed under the License is distributed on an "AS IS" BASIS, -@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem * See the License for the specific language governing permissions and -@rem * limitations under the License. -@rem */ - -@rem Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -@rem Stop hadoop hbase daemons. Run this on master node. - -setlocal - -for %%i in (%0) do ( - if not defined HBASE_BIN_PATH ( - set HBASE_BIN_PATH=%%~dpi - ) -) - -if "%HBASE_BIN_PATH:~-1%" == "\" ( - set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% -) -set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd -call %hbase-config-script% - -set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed -for /f %%i in ('%distModeCommand%') do set distMode=%%i - -if "%distMode%"=="false" ( - call %HBASE_BIN_PATH%\hbase.cmd master stop - -) else ( - if "%distMode%"=="true" ( - @echo This is not implemented yet. Stay tuned. - ) else ( - echo ERROR: Could not determine the startup mode. - ) -) - -@rem -------------- End of main script -------------- -endlocal -goto :eof \ No newline at end of file diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.sh deleted file mode 100755 index 8b81deedd8..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/stop-hbase.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -# Stop hadoop hbase daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh -. "$bin"/hbase-common.sh - -# variables needed for stop command -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid - -echo -n stopping hbase -echo "`date` Stopping hbase (via master)" >> $loglog - -nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - master stop "$@" > "$logout" 2>&1 < /dev/null & - -waitForProcessEnd `cat $pid` 'stop-master-command' - -rm -f $pid - - -# distributed == false means that the HMaster will kill ZK when it exits -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'true' ] -then - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper -fi diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/test/process_based_cluster.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/test/process_based_cluster.sh deleted file mode 100755 index eb8633f502..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/test/process_based_cluster.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# -#/** -# * Copyright The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e -u -o pipefail - -SCRIPT_NAME=${0##*/} -SCRIPT_DIR=$(cd `dirname $0` && pwd ) - -print_usage() { - cat >&2 < -Options: - --kill - Kill local process-based HBase cluster using pid files. - --show - Show HBase processes running on this machine -EOT - exit 1 -} - -show_processes() { - ps -ef | grep -P "(HRegionServer|HMaster|HQuorumPeer) start" | grep -v grep -} - -cmd_specified() { - if [ "$CMD_SPECIFIED" ]; then - echo "Only one command can be specified" >&2 - exit 1 - fi - CMD_SPECIFIED=1 -} - -list_pid_files() { - LOCAL_CLUSTER_DIR=$SCRIPT_DIR/../../target/local_cluster - LOCAL_CLUSTER_DIR=$( cd $LOCAL_CLUSTER_DIR && pwd ) - find $LOCAL_CLUSTER_DIR -name "*.pid" -} - -if [ $# -eq 0 ]; then - print_usage -fi - -IS_KILL="" -IS_SHOW="" -CMD_SPECIFIED="" - -while [ $# -ne 0 ]; do - case "$1" in - -h|--help) - print_usage ;; - --kill) - IS_KILL=1 - cmd_specified ;; - --show) - IS_SHOW=1 - cmd_specified ;; - *) - echo "Invalid option: $1" >&2 - exit 1 - esac - shift -done - -if [ "$IS_KILL" ]; then - list_pid_files | \ - while read F; do - PID=`cat $F` - echo "Killing pid $PID from file $F" - # Kill may fail but that's OK, so turn off error handling for a moment. - set +e - kill -9 $PID - set -e - done -elif [ "$IS_SHOW" ]; then - PIDS="" - for F in `list_pid_files`; do - PID=`cat $F` - if [ -n "$PID" ]; then - if [ -n "$PIDS" ]; then - PIDS="$PIDS," - fi - PIDS="$PIDS$PID" - fi - done - ps -p $PIDS -else - echo "No command specified" >&2 - exit 1 -fi - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/zookeepers.sh b/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/zookeepers.sh deleted file mode 100755 index 97bf41b605..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase-server/src/test/bin/zookeepers.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -if [ "$HBASE_MANAGES_ZK" = "" ]; then - HBASE_MANAGES_ZK=true -fi - -if [ "$HBASE_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/janusgraph-hbase-parent/janusgraph-hbase/pom.xml b/janusgraph-hbase-parent/janusgraph-hbase/pom.xml deleted file mode 100644 index 848d890015..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase/pom.xml +++ /dev/null @@ -1,172 +0,0 @@ - - 4.0.0 - - org.janusgraph - janusgraph-hbase-parent - 0.6.0-SNAPSHOT - ../pom.xml - - janusgraph-hbase - JanusGraph-HBase: Universal binary - https://janusgraph.org - - - ${basedir}/../.. - - - - - ${project.groupId} - janusgraph-hbase-core - ${project.version} - true - - - ${project.groupId} - janusgraph-hbase-10 - ${project.version} - true - - - - - - - janusgraph-release - - - - ${project.groupId} - janusgraph-hbase-core - ${project.version} - sources - true - - - ${project.groupId} - janusgraph-hbase-core - ${project.version} - javadoc - true - - - ${project.groupId} - janusgraph-hbase-10 - ${project.version} - sources - true - - - - - - - maven-assembly-plugin - - - build-hbase-source-and-javdoc-jars - package - - single - - - true - true - - ${project.basedir}/src/assembly/sources-jar.xml - ${project.basedir}/src/assembly/javadoc-jar.xml - - - - - - - - - - - - - - - maven-assembly-plugin - - - build-hbase-code-jar - package - - single - - - true - false - - ${project.basedir}/src/assembly/code-jar.xml - - - - - copy-hbase-bin - none - - - - - - maven-resources-plugin - - - filter-hbase-conf - none - - - - - - maven-clean-plugin - - - clean-hbase-files - none - - - - - - maven-jar-plugin - - - default-jar - none - - - - - - - maven-gpg-plugin - - - sign-artifacts - - - - - - maven-surefire-plugin - - - default-test - none - - - tinkerpop-test - none - - - - - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/code-jar.xml b/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/code-jar.xml deleted file mode 100644 index 994f183f7b..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/code-jar.xml +++ /dev/null @@ -1,23 +0,0 @@ - - janusgraph-hbase - / - - - jar - - - - - false - true - - - META-INF/** - - - - org.janusgraph:janusgraph-hbase-* - - - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/javadoc-jar.xml b/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/javadoc-jar.xml deleted file mode 100644 index 3ce7374120..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/javadoc-jar.xml +++ /dev/null @@ -1,23 +0,0 @@ - - javadoc - / - - - jar - - - - - false - true - - - META-INF/** - - - - ${project.groupId}:janusgraph-hbase-*:jar:javadoc:${project.version} - - - - diff --git a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/sources-jar.xml b/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/sources-jar.xml deleted file mode 100644 index 20edc78d07..0000000000 --- a/janusgraph-hbase-parent/janusgraph-hbase/src/assembly/sources-jar.xml +++ /dev/null @@ -1,23 +0,0 @@ - - sources - / - - - jar - - - - - false - true - - - META-INF/** - - - - ${project.groupId}:janusgraph-hbase-*:jar:sources:${project.version} - - - - diff --git a/janusgraph-hbase-parent/pom.xml b/janusgraph-hbase-parent/pom.xml deleted file mode 100644 index 33e1e11003..0000000000 --- a/janusgraph-hbase-parent/pom.xml +++ /dev/null @@ -1,151 +0,0 @@ - - 4.0.0 - - org.janusgraph - janusgraph - 0.6.0-SNAPSHOT - ../pom.xml - - janusgraph-hbase-parent - pom - JanusGraph-HBase: Parent Module - https://janusgraph.org - - - janusgraph-hbase-core - janusgraph-hbase-server - janusgraph-hbase-10 - janusgraph-hbase - - - - -Dhadoop.version=${hadoop.version} -Dhadoop.profile=${hadoop.profile} ${janusgraph.hbase.mvn.opts.extra} - - -Xms256m -Xmx1280m -XX:+HeapDumpOnOutOfMemoryError -ea ${test.extra.jvm.opts} - -Xms256m -Xmx768m -ea -XX:+HeapDumpOnOutOfMemoryError ${test.extra.jvm.opts} - ${project.parent.basedir} - - - - - - - maven-resources-plugin - - - filter-hbase-conf - process-test-resources - - copy-resources - - - ${project.basedir}/conf - - $MAVEN{*} - - - - ${project.basedir}/../common/test/conf/ - true - - - - - - - - maven-surefire-plugin - - - ${top.level.basedir}/janusgraph-hbase-parent - - ${test.excluded.groups} - - - - maven-dependency-plugin - - - generate-cached-classpath - generate-test-resources - - build-classpath - - - ${project.basedir}/target/cached_classpath.txt - - - - unpack-common-test-classes - process-test-classes - - unpack - - - - - ${project.groupId} - janusgraph-hbase-core - ${project.version} - tests - ${project.build.directory}/test-classes - - - - - - - - - maven-assembly-plugin - - - copy-hbase-bin - process-test-resources - - single - - - false - / - false - ${project.basedir}/bin - - ${project.basedir}/../common/test/binassembly.xml - - - - - - - - maven-clean-plugin - - - clean-hbase-files - clean - - clean - - - true - - - bin/ - - - conf/ - - - logs/ - - - - - - - - - - diff --git a/janusgraph-hbase/docker/Dockerfile b/janusgraph-hbase/docker/Dockerfile new file mode 100644 index 0000000000..9b38c45b73 --- /dev/null +++ b/janusgraph-hbase/docker/Dockerfile @@ -0,0 +1,56 @@ +# Copyright 2021 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:buster as builder + +ARG HBASE_VERSION=2.2.7 +ARG HBASE_DIST="http://ftp.halifax.rwth-aachen.de/apache/hbase" + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl=7.64.0-4+deb10u2 + +RUN curl -SL ${HBASE_DIST}/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz | tar -x -z && mv hbase-${HBASE_VERSION} /opt/hbase +WORKDIR /opt/hbase +RUN rm -rf ./docs ./LEGAL ./*.txt ./*/*.cmd + +COPY ./hbase-site.xml /opt/hbase/conf/hbase-site.xml +COPY ./hbase-policy.xml /opt/hbase/conf/hbase-policy.xml +COPY ./zoo.cfg /opt/hbase/conf/zoo.cfg + +FROM openjdk:8-jre-buster + +ARG HBASE_UID=1000 +ARG HBASE_GID=1000 + +ENV LOGS_DIR=/data/logs + +COPY --from=builder /opt/hbase/ /opt/hbase/ + +RUN ln -sf /opt/hbase/bin/* /usr/bin/ +RUN mkdir -p $LOGS_DIR /data/hbase /data/run + +WORKDIR /opt/hbase + +EXPOSE 2181 +EXPOSE 16000 +EXPOSE 16010 +EXPOSE 16020 +EXPOSE 16030 + +CMD ["/usr/bin/hbase", "master", "start"] + +RUN groupadd -g ${HBASE_GID} hbase && \ + useradd -s /bin/bash hbase -u ${HBASE_UID} -g ${HBASE_GID} && \ + chown -R ${HBASE_UID}:${HBASE_GID} /data +USER hbase diff --git a/janusgraph-hbase-parent/common/test/conf/hbase-policy.xml b/janusgraph-hbase/docker/hbase-policy.xml similarity index 100% rename from janusgraph-hbase-parent/common/test/conf/hbase-policy.xml rename to janusgraph-hbase/docker/hbase-policy.xml diff --git a/janusgraph-hbase/docker/hbase-site.xml b/janusgraph-hbase/docker/hbase-site.xml new file mode 100644 index 0000000000..98d4a78760 --- /dev/null +++ b/janusgraph-hbase/docker/hbase-site.xml @@ -0,0 +1,33 @@ + + + + + hbase.zookeeper.quorum + localhost + + + hbase.rootdir + file:////data/hbase + + + + hbase.master.info.bindAddress + 0.0.0.0 + + + + hbase.regionserver.info.bindAddress + 0.0.0.0 + + + + hbase.master.ipc.address + 0.0.0.0 + + + + hbase.regionserver.ipc.address + 0.0.0.0 + + + diff --git a/janusgraph-hbase/docker/zoo.cfg b/janusgraph-hbase/docker/zoo.cfg new file mode 100644 index 0000000000..c716a49fc7 --- /dev/null +++ b/janusgraph-hbase/docker/zoo.cfg @@ -0,0 +1,15 @@ +# Copyright 2021 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.1=localhost:2181 diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/pom.xml b/janusgraph-hbase/pom.xml similarity index 59% rename from janusgraph-hbase-parent/janusgraph-hbase-core/pom.xml rename to janusgraph-hbase/pom.xml index d3366745d6..36b7716cb0 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/pom.xml +++ b/janusgraph-hbase/pom.xml @@ -2,20 +2,35 @@ 4.0.0 org.janusgraph - janusgraph-hbase-parent + janusgraph 0.6.0-SNAPSHOT - ../pom.xml - janusgraph-hbase-core - JanusGraph-HBase: Version-independent Core + janusgraph-hbase + JanusGraph-HBase: Distributed Graph Database https://janusgraph.org 2.0 - ${basedir}/../.. + 1000 + 1000 + ${basedir}/.. + ${basedir}/target + + + + org.slf4j + slf4j-log4j12 + true + + + ch.qos.logback + logback-classic + + + org.janusgraph janusgraph-core @@ -40,26 +55,55 @@ tests test + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + net.java.dev.jna + jna + ${jna.version} + test + - src/main/java + ${basedir}/target + + + ${basedir}/src/main/resources + true + + - maven-resources-plugin + maven-surefire-plugin + + + file:${project.build.testOutputDirectory}/log4j.properties + ${hbase.docker.version} + ${hbase.docker.uid} + ${hbase.docker.gid} + ${test.hbase.targetdir} + + - filter-hbase-conf + default-test none - - - - maven-deploy-plugin - - default-deploy - none + hbase-test + + test + + test @@ -68,33 +112,23 @@ pack-test-jar - package + + prepare-package test-jar - - maven-surefire-plugin - - - - default-test - none - - - tinkerpop-test - none - - - hbase1 + + ${hbase1.version} + org.apache.hbase @@ -118,6 +152,9 @@ hbase2 + + ${hbase2.version} + !hbase.profile @@ -145,6 +182,10 @@ junit junit + + org.apache.hadoop + hadoop-annotations + true diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/com/google/common/base/Stopwatch.java b/janusgraph-hbase/src/main/java/com/google/common/base/Stopwatch.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/com/google/common/base/Stopwatch.java rename to janusgraph-hbase/src/main/java/com/google/common/base/Stopwatch.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/com/google/common/io/Closeables.java b/janusgraph-hbase/src/main/java/com/google/common/io/Closeables.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/com/google/common/io/Closeables.java rename to janusgraph-hbase/src/main/java/com/google/common/io/Closeables.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/AdminMask.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/AdminMask.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/AdminMask.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/AdminMask.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/ConnectionMask.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/ConnectionMask.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/ConnectionMask.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/ConnectionMask.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HBaseAdmin1_0.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseAdmin1_0.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HBaseAdmin1_0.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseAdmin1_0.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat1_0.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat1_0.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat1_0.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompat1_0.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java similarity index 97% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java index 399912d187..de68f9e1bb 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java +++ b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseCompatLoader.java @@ -28,7 +28,7 @@ public class HBaseCompatLoader { "org.janusgraph.diskstorage.hbase.HBaseCompat1_0"; private static final String[] HBASE_SUPPORTED_VERSIONS = - new String[] { "1.2", "1.3", "1.4", "1.6", "2.0", "2.1" }; + new String[] { "1.2", "1.3", "1.4", "1.6", "2.0", "2.1", "2.2" }; private static HBaseCompat cachedCompat; diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseKeyColumnValueStore.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseKeyColumnValueStore.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseKeyColumnValueStore.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseKeyColumnValueStore.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseStoreManager.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseStoreManager.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseStoreManager.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseStoreManager.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseTransaction.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseTransaction.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/HBaseTransaction.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HBaseTransaction.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HConnection1_0.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HConnection1_0.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HConnection1_0.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HConnection1_0.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HTable1_0.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HTable1_0.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-10/src/main/java/org/janusgraph/diskstorage/hbase/HTable1_0.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/HTable1_0.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/TableMask.java b/janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/TableMask.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/diskstorage/hbase/TableMask.java rename to janusgraph-hbase/src/main/java/org/janusgraph/diskstorage/hbase/TableMask.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/HBaseHadoopStoreManager.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/HBaseHadoopStoreManager.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/HBaseHadoopStoreManager.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/HBaseHadoopStoreManager.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/HBaseMapReduceIndexJobsUtils.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/HBaseMapReduceIndexJobsUtils.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/HBaseMapReduceIndexJobsUtils.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/HBaseMapReduceIndexJobsUtils.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/config/HBaseAuthHelper.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/config/HBaseAuthHelper.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/config/HBaseAuthHelper.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/config/HBaseAuthHelper.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryInputFormat.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryInputFormat.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryInputFormat.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryInputFormat.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryRecordReader.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryRecordReader.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryRecordReader.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseBinaryRecordReader.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseInputFormat.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseInputFormat.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseInputFormat.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseInputFormat.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotBinaryInputFormat.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotBinaryInputFormat.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotBinaryInputFormat.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotBinaryInputFormat.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotInputFormat.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotInputFormat.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotInputFormat.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/formats/hbase/HBaseSnapshotInputFormat.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/scan/HBaseHadoopScanRunner.java b/janusgraph-hbase/src/main/java/org/janusgraph/hadoop/scan/HBaseHadoopScanRunner.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/main/java/org/janusgraph/hadoop/scan/HBaseHadoopScanRunner.java rename to janusgraph-hbase/src/main/java/org/janusgraph/hadoop/scan/HBaseHadoopScanRunner.java diff --git a/janusgraph-hbase/src/test/java/org/janusgraph/HBaseContainer.java b/janusgraph-hbase/src/test/java/org/janusgraph/HBaseContainer.java new file mode 100644 index 0000000000..6f97546c0d --- /dev/null +++ b/janusgraph-hbase/src/test/java/org/janusgraph/HBaseContainer.java @@ -0,0 +1,176 @@ +// Copyright 2021 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.janusgraph.core.JanusGraphException; +import org.janusgraph.diskstorage.BackendException; +import org.janusgraph.diskstorage.TemporaryBackendException; +import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; +import org.janusgraph.diskstorage.configuration.WriteConfiguration; +import org.janusgraph.diskstorage.hbase.AdminMask; +import org.janusgraph.diskstorage.hbase.ConnectionMask; +import org.janusgraph.diskstorage.hbase.HBaseCompat; +import org.janusgraph.diskstorage.hbase.HBaseCompatLoader; +import org.janusgraph.diskstorage.hbase.HBaseStoreManager; +import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.database.idassigner.placement.SimpleBulkPlacementStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class HBaseContainer extends GenericContainer { + private static final Logger logger = LoggerFactory.getLogger(HBaseContainer.class); + + public static final String HBASE_TARGET_DIR = "test.hbase.targetdir"; + private static final String DEFAULT_VERSION = "2.2.7"; + private static final String DEFAULT_UID = "1000"; + private static final String DEFAULT_GID = "1000"; + + private final HBaseCompat compat; + + private static String getVersion() { + String property = System.getProperty("hbase.docker.version"); + if (StringUtils.isNotEmpty(property)) + return property; + return DEFAULT_VERSION; + } + + private static String getUid() { + String property = System.getProperty("hbase.docker.uid"); + if (StringUtils.isNotEmpty(property)) + return property; + return DEFAULT_UID; + } + + private static String getGid() { + String property = System.getProperty("hbase.docker.gid"); + if (StringUtils.isNotEmpty(property)) + return property; + return DEFAULT_GID; + } + + private static String getTargetDir() { + String property = System.getProperty(HBASE_TARGET_DIR); + if (StringUtils.isNotEmpty(property)) + return property; + return Paths.get(System.getProperty("user.dir"), "target").toString(); + } + + public HBaseContainer() { + this(false); + } + + public HBaseContainer(boolean mountRoot) { + super(new ImageFromDockerfile() + .withFileFromPath(".", Paths.get("docker")) + .withBuildArg("HBASE_VERSION", getVersion()) + .withBuildArg("HBASE_UID", getUid()) + .withBuildArg("HBASE_GID", getGid())); + addFixedExposedPort(2181, 2181); + addFixedExposedPort(16000, 16000); + addFixedExposedPort(16010, 16010); + addFixedExposedPort(16020, 16020); + addFixedExposedPort(16030, 16030); + + if (mountRoot){ + try { + Files.createDirectories(getHBaseRootdir()); + } catch (IOException e) { + logger.warn("failed to create folder", e); + throw new JanusGraphException(e); + } + addFileSystemBind(getHBaseRootdir().toString(), "/data/hbase", BindMode.READ_WRITE); + } + + withCreateContainerCmdModifier(createContainerCmd -> { + createContainerCmd + .withHostName("localhost"); + }); + waitingFor(Wait.forLogMessage(".*Master has completed initialization.*", 1)); + compat = HBaseCompatLoader.getCompat(null); + } + + public Path getHBaseRootdir() { + return Paths.get(getTargetDir(), "hbase-root"); + } + + private ConnectionMask createConnectionMask() throws IOException { + Configuration entries = HBaseConfiguration.create(); + entries.set("hbase.zookeeper.quorum", "localhost"); + return compat.createConnection(entries); + } + + /** + * Create a snapshot for a table. + * + * @param snapshotName + * @param table + * @throws BackendException + */ + public synchronized void createSnapshot(String snapshotName, String table) + throws BackendException { + try (ConnectionMask hc = createConnectionMask(); + AdminMask admin = hc.getAdmin()) { + admin.snapshot(snapshotName, table); + } catch (Exception e) { + logger.warn("Create HBase snapshot failed", e); + throw new TemporaryBackendException("Create HBase snapshot failed", e); + } + } + + /** + * Delete a snapshot. + * + * @param snapshotName + * @throws IOException + */ + public synchronized void deleteSnapshot(String snapshotName) throws IOException { + try (ConnectionMask hc = createConnectionMask(); + AdminMask admin = hc.getAdmin()) { + admin.deleteSnapshot(snapshotName); + } + } + + public WriteConfiguration getWriteConfiguration() { + return getModifiableConfiguration().getConfiguration(); + } + + public ModifiableConfiguration getModifiableConfiguration() { + return getNamedConfiguration(null, null); + } + + public ModifiableConfiguration getNamedConfiguration(String tableName, String graphName) { + ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); + config.set(GraphDatabaseConfiguration.STORAGE_BACKEND, "hbase"); + if (StringUtils.isNotEmpty(tableName)) config.set(HBaseStoreManager.HBASE_TABLE, tableName); + if (StringUtils.isNotEmpty(graphName)) config.set(GraphDatabaseConfiguration.GRAPH_NAME, graphName); + config.set(GraphDatabaseConfiguration.TIMESTAMP_PROVIDER, HBaseStoreManager.PREFERRED_TIMESTAMPS); + config.set(GraphDatabaseConfiguration.STORAGE_HOSTS, new String[]{"localhost"}); + config.set(SimpleBulkPlacementStrategy.CONCURRENT_PARTITIONS, 1); + config.set(GraphDatabaseConfiguration.DROP_ON_CLEAR, false); + return config; + } +} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java similarity index 81% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java index 57b2d11bda..6bc6743b81 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphComputerProvider.java @@ -14,12 +14,9 @@ package org.janusgraph.blueprints; -import org.janusgraph.HBaseStorageSetup; +import org.apache.tinkerpop.gremlin.GraphProvider; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.graphdb.olap.computer.FulgoraGraphComputer; -import org.apache.tinkerpop.gremlin.GraphProvider; - -import java.io.IOException; /** * @author Matthias Broecheler (me@matthiasb.com) @@ -29,13 +26,8 @@ public class HBaseGraphComputerProvider extends AbstractJanusGraphComputerProvid @Override public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class test, String testMethodName) { - try { - HBaseStorageSetup.startHBase(); - } catch (IOException e) { - throw new RuntimeException(e); - } ModifiableConfiguration config = super.getJanusGraphConfiguration(graphName, test, testMethodName); - config.setAll(HBaseStorageSetup.getHBaseConfiguration(graphName).getAll()); + config.setAll(HBaseGraphProvider.HBASE_CONTAINER.getNamedConfiguration(graphName, "").getAll()); return config; } diff --git a/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java new file mode 100644 index 0000000000..fb4035a951 --- /dev/null +++ b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseGraphProvider.java @@ -0,0 +1,66 @@ +// Copyright 2017 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.blueprints; + +import org.janusgraph.HBaseContainer; +import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.ServerSocket; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * @author Matthias Broecheler (me@matthiasb.com) + */ +public class HBaseGraphProvider extends AbstractJanusGraphProvider { + private static final Logger logger = LoggerFactory.getLogger(HBaseGraphProvider.class); + public static final HBaseContainer HBASE_CONTAINER; + + static { + waitForBindablePort(2181); + waitForBindablePort(16000); + HBASE_CONTAINER = new HBaseContainer(); + HBASE_CONTAINER.start(); + } + + private static void waitForBindablePort(int port) { + boolean canBindPort = false; + do { + try (ServerSocket serverSocket = new ServerSocket(port)) { + assertNotNull(serverSocket); + assertEquals(serverSocket.getLocalPort(), port); + serverSocket.close(); + canBindPort = true; + continue; + } catch (IOException e) { + logger.warn("can't bind port", e); + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.warn("can't sleep", e); + } + } while (!canBindPort); + } + + @Override + public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class test, String testMethodName) { + return HBASE_CONTAINER.getNamedConfiguration(graphName, ""); + } +} diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java similarity index 79% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java index bc70b40ac3..f1eb00ed3f 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/HBaseMultiQueryGraphProvider.java @@ -14,25 +14,16 @@ package org.janusgraph.blueprints; -import org.janusgraph.HBaseStorageSetup; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; -import java.io.IOException; - /** * @author Ted Wilmes (twilmes@gmail.com) */ public class HBaseMultiQueryGraphProvider extends AbstractJanusGraphProvider { - @Override public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class test, String testMethodName) { - try { - HBaseStorageSetup.startHBase(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return HBaseStorageSetup.getHBaseConfiguration(graphName) + return HBaseGraphProvider.HBASE_CONTAINER.getNamedConfiguration(graphName, "") .set(GraphDatabaseConfiguration.USE_MULTIQUERY, true); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseComputerTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseComputerTest.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseComputerTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseComputerTest.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseMultiQueryProcessTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseMultiQueryProcessTest.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseMultiQueryProcessTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseMultiQueryProcessTest.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseProcessTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseProcessTest.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/process/HBaseProcessTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/process/HBaseProcessTest.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/structure/HBaseStructureTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/blueprints/structure/HBaseStructureTest.java similarity index 100% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/blueprints/structure/HBaseStructureTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/blueprints/structure/HBaseStructureTest.java diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java similarity index 77% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java index 4e39e6781f..a49c9f5824 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseDistributedStoreManagerTest.java @@ -14,27 +14,23 @@ package org.janusgraph.diskstorage.hbase; -import java.io.IOException; - +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; +import org.janusgraph.diskstorage.DistributedStoreManagerTest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.BeforeAll; -import org.janusgraph.HBaseStorageSetup; -import org.janusgraph.diskstorage.DistributedStoreManagerTest; - +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseDistributedStoreManagerTest extends DistributedStoreManagerTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @BeforeEach public void setUp() throws BackendException { - manager = new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + manager = new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); store = manager.openDatabase("distributedStoreTest"); } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java similarity index 74% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java index c3eb53c03c..d3f94873ee 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseIDAuthorityTest.java @@ -14,23 +14,21 @@ package org.janusgraph.diskstorage.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.IDAuthorityTest; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.junit.jupiter.api.BeforeAll; -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseIDAuthorityTest extends IDAuthorityTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); public KeyColumnValueStoreManager openStorageManager() throws BackendException { - return new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + return new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java similarity index 76% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java index 892d8006ad..d67a65dc5c 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLockStoreTest.java @@ -14,24 +14,21 @@ package org.janusgraph.diskstorage.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.LockKeyColumnValueStoreTest; import org.janusgraph.diskstorage.configuration.Configuration; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseLockStoreTest extends LockKeyColumnValueStoreTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); public KeyColumnValueStoreManager openStorageManager(int idx, Configuration configuration) throws BackendException { - return new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + return new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java similarity index 74% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java index 7948f2337c..c0520d9516 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseLogTest.java @@ -14,26 +14,22 @@ package org.janusgraph.diskstorage.hbase; -import java.io.IOException; - +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; - -import org.junit.jupiter.api.BeforeAll; - -import org.janusgraph.HBaseStorageSetup; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.janusgraph.diskstorage.log.KCVSLogTest; -public class HBaseLogTest extends KCVSLogTest { +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } +@Testcontainers +public class HBaseLogTest extends KCVSLogTest { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public KeyColumnValueStoreManager openStorageManager() throws BackendException { - return new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + return new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java similarity index 74% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java index c0f6476653..b74773bcf8 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseMultiWriteStoreTest.java @@ -14,23 +14,19 @@ package org.janusgraph.diskstorage.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.MultiWriteKeyColumnValueStoreTest; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; - +@Testcontainers public class HBaseMultiWriteStoreTest extends MultiWriteKeyColumnValueStoreTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer _hBaseContainer = new HBaseContainer(); public KeyColumnValueStoreManager openStorageManager() throws BackendException { - return new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + return new HBaseStoreManager(_hBaseContainer.getModifiableConfiguration()); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java similarity index 88% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java index 861ec8aa21..9134a6d832 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerConfigTest.java @@ -14,17 +14,13 @@ package org.janusgraph.diskstorage.hbase; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; -import java.io.StringWriter; - import org.apache.log4j.Appender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.apache.log4j.WriterAppender; -import org.janusgraph.HBaseStorageSetup; + +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.configuration.BasicConfiguration; import org.janusgraph.diskstorage.configuration.ConfigElement; @@ -34,22 +30,20 @@ import org.janusgraph.diskstorage.util.time.TimestampProviders; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; import org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -public class HBaseStoreManagerConfigTest { +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; - @BeforeAll - public static void startHBase() throws IOException, BackendException { - HBaseStorageSetup.startHBase(); - } +import java.io.StringWriter; - @AfterAll - public static void stopHBase() { - // No op. HBase stopped by shutdown hook registered by startHBase(). +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; - } +@Testcontainers +public class HBaseStoreManagerConfigTest { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Test public void testShortCfNames() throws Exception { @@ -61,7 +55,7 @@ public void testShortCfNames() throws Exception { log.addAppender(appender); // Open the HBaseStoreManager and store with default SHORT_CF_NAMES true. - WriteConfiguration config = HBaseStorageSetup.getHBaseGraphConfiguration(); + WriteConfiguration config = hBaseContainer.getWriteConfiguration(); HBaseStoreManager manager = new HBaseStoreManager(new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, config, BasicConfiguration.Restriction.NONE)); KeyColumnValueStore store = manager.openDatabase(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME); @@ -102,7 +96,7 @@ public void testHBaseTimestampProvider() throws BackendException { @Test public void testHBaseStoragePort() throws BackendException { - WriteConfiguration config = HBaseStorageSetup.getHBaseGraphConfiguration(); + WriteConfiguration config = hBaseContainer.getWriteConfiguration(); config.set(ConfigElement.getPath(GraphDatabaseConfiguration.STORAGE_PORT), 2000); HBaseStoreManager manager = new HBaseStoreManager(new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, config, BasicConfiguration.Restriction.NONE)); diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java similarity index 94% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java index ee1e41a68d..aa191af630 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreManagerMutationTest.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Pair; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.Entry; import org.janusgraph.diskstorage.EntryMetaData; import org.janusgraph.diskstorage.KeyColumnValueStoreUtil; @@ -35,11 +35,16 @@ import org.janusgraph.diskstorage.util.StaticArrayBuffer; import org.janusgraph.diskstorage.util.StaticArrayEntry; import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +@Testcontainers public class HBaseStoreManagerMutationTest { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Test public void testKCVMutationToPuts() throws Exception { @@ -88,7 +93,7 @@ public void testKCVMutationToPuts() throws Exception { rowkeyMutationMap.put(rowkey, new KCVMutation(additions, deletions)); } storeMutationMap.put("store1", rowkeyMutationMap); - HBaseStoreManager manager = new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + HBaseStoreManager manager = new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); final Map, Delete>> commandsPerRowKey = manager.convertToCommands(storeMutationMap, 0, 0); @@ -156,7 +161,7 @@ public void testMutationToPutsTTL() throws Exception{ deletions.add(e); rowkeyMutationMap.put(rowkey, new KCVMutation(additions, deletions)); storeMutationMap.put("store1", rowkeyMutationMap); - HBaseStoreManager manager = new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration()); + HBaseStoreManager manager = new HBaseStoreManager(hBaseContainer.getModifiableConfiguration()); final Map, Delete>> commandsPerRowKey = manager.convertToCommands(storeMutationMap, 0, 0); Pair, Delete> commands = commandsPerRowKey.values().iterator().next(); diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java similarity index 83% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java index 7370e6dbe4..113f7a0317 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/diskstorage/hbase/HBaseStoreTest.java @@ -14,27 +14,24 @@ package org.janusgraph.diskstorage.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.KeyColumnValueStoreTest; import org.janusgraph.diskstorage.configuration.BasicConfiguration; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.IOException; - +@Testcontainers public class HBaseStoreTest extends KeyColumnValueStoreTest { - - @BeforeAll - public static void startHBase() throws IOException, BackendException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); public HBaseStoreManager openStorageManager(ModifiableConfiguration config) throws BackendException { return new HBaseStoreManager(new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, config.getConfiguration(), BasicConfiguration.Restriction.NONE)); @@ -49,7 +46,7 @@ public HBaseStoreManager openStorageManager(String tableName) throws BackendExce } public HBaseStoreManager openStorageManager(String tableName, String graphName) throws BackendException { - return new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration(tableName, graphName)); + return new HBaseStoreManager(hBaseContainer.getNamedConfiguration(tableName, graphName)); } @Test @@ -59,7 +56,7 @@ public void testGetKeysWithKeyRange(TestInfo testInfo) throws Exception { @Override public HBaseStoreManager openStorageManagerForClearStorageTest() throws Exception { - return openStorageManager(HBaseStorageSetup.getHBaseConfiguration().set(GraphDatabaseConfiguration.DROP_ON_CLEAR, true)); + return openStorageManager(hBaseContainer.getModifiableConfiguration().set(GraphDatabaseConfiguration.DROP_ON_CLEAR, true)); } @Test diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java similarity index 74% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java index 75c0255af4..f1343de8cb 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphConcurrentTest.java @@ -14,23 +14,20 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphConcurrentTest; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseGraphConcurrentTest extends JanusGraphConcurrentTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java similarity index 75% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java index 13249679a4..e25981f3dc 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphPerformanceMemoryTest.java @@ -14,23 +14,21 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphPerformanceMemoryTest; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseGraphPerformanceMemoryTest extends JanusGraphPerformanceMemoryTest { - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java similarity index 79% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java index 5819aa8244..33bfa39c1f 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseGraphTest.java @@ -14,25 +14,23 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphTest; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; - +@Testcontainers public class HBaseGraphTest extends JanusGraphTest { - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } @Override @Test @Disabled("HBase does not support retrieving cell TTL by client") diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java similarity index 73% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java index 4addf89a7a..535b8ea588 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOLAPTest.java @@ -14,23 +14,21 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.olap.OLAPTest; -import org.junit.jupiter.api.BeforeAll; -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseOLAPTest extends OLAPTest { - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java similarity index 74% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java index 6911d221a9..58968676dd 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBaseOperationCountingTest.java @@ -14,30 +14,29 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphOperationCountingTest; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; +import org.junit.jupiter.api.Disabled; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; /** * @author Matthias Broecheler (me@matthiasb.com) */ +@Testcontainers public class HBaseOperationCountingTest extends JanusGraphOperationCountingTest { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getBaseConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); - } - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); + return hBaseContainer.getWriteConfiguration(); } @Override + @Disabled public void testCacheConcurrency() { //Don't run this test; } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java similarity index 76% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java rename to janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java index f9465c06ed..7389246144 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/graphdb/hbase/HBasePartitionGraphTest.java @@ -14,27 +14,24 @@ package org.janusgraph.graphdb.hbase; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.graphdb.JanusGraphPartitionGraphTest; -import org.junit.jupiter.api.BeforeAll; - -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; /** * @author Matthias Broecheler (me@matthiasb.com) */ +@Testcontainers public class HBasePartitionGraphTest extends JanusGraphPartitionGraphTest { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getBaseConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); - } - - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java similarity index 57% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java rename to janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java index 47dc8e9e12..3021a33dd2 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseIndexManagementIT.java @@ -14,30 +14,20 @@ package org.janusgraph.hadoop; -import org.janusgraph.HBaseStorageSetup; +import org.janusgraph.HBaseContainer; import org.janusgraph.diskstorage.configuration.WriteConfiguration; -import org.apache.hadoop.hbase.util.VersionInfo; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import java.io.IOException; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +@Testcontainers public class HBaseIndexManagementIT extends AbstractIndexManagementIT { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } - @BeforeAll - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } - - @AfterAll - public static void stopHBase() { - // Workaround for https://issues.apache.org/jira/browse/HBASE-10312 - if (VersionInfo.getVersion().startsWith("0.96")) - HBaseStorageSetup.killIfRunning(); - } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java similarity index 80% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java rename to janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java index 9fda25fd96..5878415704 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java @@ -14,31 +14,26 @@ package org.janusgraph.hadoop; -import org.janusgraph.HBaseStorageSetup; -import org.janusgraph.diskstorage.BackendException; -import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; + +import org.janusgraph.HBaseContainer; +import org.janusgraph.diskstorage.configuration.WriteConfiguration; + +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +@Testcontainers public class HBaseInputFormatIT extends AbstractInputFormatIT { - - @BeforeAll - public static void startHBase() throws IOException, BackendException { - HBaseStorageSetup.startHBase(); - } - - @AfterAll - public static void stopHBase() { - } + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(); protected Graph getGraph() throws IOException, ConfigurationException { final PropertiesConfiguration config = new PropertiesConfiguration("target/test-classes/hbase-read.properties"); @@ -51,6 +46,6 @@ protected Graph getGraph() throws IOException, ConfigurationException { @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java similarity index 91% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java rename to janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java index e393852558..f5813afc5f 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java +++ b/janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseSnapshotInputFormatIT.java @@ -14,63 +14,62 @@ package org.janusgraph.hadoop; -import org.janusgraph.HBaseStorageSetup; -import org.janusgraph.core.Cardinality; -import org.janusgraph.core.JanusGraphVertex; -import org.janusgraph.diskstorage.BackendException; -import org.janusgraph.diskstorage.configuration.WriteConfiguration; -import org.janusgraph.example.GraphOfTheGodsFactory; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.process.computer.Computer; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; +import org.janusgraph.HBaseContainer; +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.JanusGraphVertex; +import org.janusgraph.diskstorage.configuration.WriteConfiguration; +import org.janusgraph.example.GraphOfTheGodsFactory; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * This test suite contains the same tests as in HBaseInputFormatIT and AbstractInputFormatIT, but * takes HBase snapshots of the graph table during the tests. The snapshots are used by * HBaseSnapshotInputFormat. */ +@Testcontainers public class HBaseSnapshotInputFormatIT extends AbstractInputFormatIT { + @Container + public static final HBaseContainer hBaseContainer = new HBaseContainer(true); // Used by this test only. Need to be consistent with hbase-read-snapshot.properties private final String table = "janusgraph"; private final String snapshotName = "janusgraph-snapshot"; - @BeforeAll - public static void startHBase() throws IOException, BackendException { - HBaseStorageSetup.startHBase(); - } - - @AfterAll - public static void stopHBase() { - } @AfterEach @Override public void tearDown() throws Exception { super.tearDown(); - HBaseStorageSetup.deleteSnapshot(snapshotName); + hBaseContainer.deleteSnapshot(snapshotName); } @Test @@ -79,7 +78,7 @@ public void testReadGraphOfTheGods() throws Exception { GraphOfTheGodsFactory.load(graph, null, true); assertEquals(12L, (long) graph.traversal().V().count().next()); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); Graph g = getGraph(); GraphTraversalSource t = g.traversal().withComputer(SparkGraphComputer.class); @@ -112,7 +111,7 @@ public void testReadWideVertexWithManyProperties() throws Exception { assertEquals(Integer.toString(i), valuesOnP.get(i).toString()); } // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); Graph g = getGraph(); GraphTraversalSource t = g.traversal().withComputer(SparkGraphComputer.class); @@ -145,7 +144,7 @@ public void testReadSelfEdge() throws Exception { assertEquals(3L, sky.query().direction(Direction.BOTH).edgeCount()); graph.tx().commit(); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); // Read the new edge using the inputformat Graph g = getGraph(); @@ -163,7 +162,7 @@ public void testReadSelfEdge() throws Exception { public void testGeoshapeGetValues() throws Exception { GraphOfTheGodsFactory.load(graph, null, true); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); // Read geoshape using the inputformat Graph g = getGraph(); @@ -182,7 +181,7 @@ public void testReadGraphOfTheGodsWithEdgeFiltering() throws Exception { GraphOfTheGodsFactory.load(graph, null, true); assertEquals(17L, (long) graph.traversal().E().count().next()); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); // Read graph filtering out "battled" edges. Graph g = getGraph(); @@ -199,7 +198,7 @@ public void testGraphWithIsolatedVertices() throws Exception { graph.addVertex(key); graph.tx().commit(); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); // Read graph using the inputformat. Graph g = getGraph(); @@ -215,7 +214,7 @@ public void testSchemaVerticesAreSkipped() throws Exception { mgmt.makeEdgeLabel("e").make(); finishSchema(); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); // Read graph using the inputformat. Graph g = getGraph(); @@ -254,7 +253,7 @@ public void testReadWithMetaProperties() throws Exception { .properties("meta_property").count().next()); // Take a snapshot of the graph table - HBaseStorageSetup.createSnapshot(snapshotName, table); + hBaseContainer.createSnapshot(snapshotName, table); Graph g = getGraph(); t = g.traversal().withComputer(SparkGraphComputer.class); @@ -272,12 +271,12 @@ protected Graph getGraph() throws IOException, ConfigurationException { config.setProperty("gremlin.hadoop.outputLocation", outDir); // Set the hbase.rootdir property. This is needed by HBaseSnapshotInputFormat. config.setProperty("janusgraphmr.ioformat.conf.storage.hbase.ext.hbase.rootdir", - HBaseStorageSetup.getHBaseRootdir()); + hBaseContainer.getHBaseRootdir().toString()); return GraphFactory.open(config); } @Override public WriteConfiguration getConfiguration() { - return HBaseStorageSetup.getHBaseGraphConfiguration(); + return hBaseContainer.getWriteConfiguration(); } } diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read-snapshot.properties b/janusgraph-hbase/src/test/resources/hbase-read-snapshot.properties similarity index 94% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read-snapshot.properties rename to janusgraph-hbase/src/test/resources/hbase-read-snapshot.properties index 87e03e9dfc..5c25e6ad2b 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read-snapshot.properties +++ b/janusgraph-hbase/src/test/resources/hbase-read-snapshot.properties @@ -20,12 +20,14 @@ gremlin.hadoop.jarsInDistributedCache=true gremlin.hadoop.inputLocation=none gremlin.hadoop.outputLocation=target/output janusgraphmr.ioformat.conf.storage.backend=hbase +janusgraphmr.ioformat.conf.storage.hostname=localhost janusgraphmr.ioformat.conf.storage.hbase.snapshot-name=janusgraph-snapshot janusgraphmr.ioformat.conf.storage.hbase.snapshot-restore-dir=target/output #################################### # SparkGraphComputer Configuration # #################################### spark.master=local[4] +spark.driver.host=127.0.0.1 spark.executor.memory=1g spark.serializer=org.apache.spark.serializer.KryoSerializer spark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read.properties b/janusgraph-hbase/src/test/resources/hbase-read.properties similarity index 97% rename from janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read.properties rename to janusgraph-hbase/src/test/resources/hbase-read.properties index 3c4fa829b0..3f06e919af 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/resources/hbase-read.properties +++ b/janusgraph-hbase/src/test/resources/hbase-read.properties @@ -25,6 +25,7 @@ janusgraphmr.ioformat.conf.storage.hostname=localhost # SparkGraphComputer Configuration # #################################### spark.master=local[4] +spark.driver.host=127.0.0.1 spark.executor.memory=1g spark.serializer=org.apache.spark.serializer.KryoSerializer spark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator diff --git a/janusgraph-hbase-parent/common/test/resources/log4j.properties b/janusgraph-hbase/src/test/resources/log4j.properties similarity index 60% rename from janusgraph-hbase-parent/common/test/resources/log4j.properties rename to janusgraph-hbase/src/test/resources/log4j.properties index 3f495c8c10..b8f0e78f14 100644 --- a/janusgraph-hbase-parent/common/test/resources/log4j.properties +++ b/janusgraph-hbase/src/test/resources/log4j.properties @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A1 is a FileAppender. +# A1 is set to be a FileAppender. log4j.appender.A1=org.apache.log4j.FileAppender log4j.appender.A1.File=target/test.log log4j.appender.A1.Threshold=ALL @@ -31,18 +31,3 @@ log4j.appender.A2.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p #log4j.rootLogger=INFO, A1, A2 log4j.rootLogger=ERROR, A1 -# Restrict some of JanusGraph's dependencies to INFO and scarier. -# These restrictions are useful when reducing the severity threshold -# setting on one of the appenders below INFO. -log4j.logger.org.apache.cassandra=INFO -log4j.logger.org.apache.hadoop=INFO -log4j.logger.org.apache.zookeeper=INFO -# Disable all messages from ExpectedValueCheckingTransaction. The point is to -# suppress scary-looking ERROR messages that are deliberately induced by -# LockKeyColumnValueStoreTest. -log4j.logger.org.janusgraph.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction=OFF - -# This DEBUG line produces more than 100 MB of output per test shim -#log4j.logger.org.janusgraph.diskstorage.hbase=DEBUG -log4j.logger.org.janusgraph.diskstorage.IDAllocationTest=DEBUG -log4j.logger.org.janusgraph.HBaseStorageSetup=DEBUG diff --git a/mkdocs.yml b/mkdocs.yml index 5f6b51308d..f1fb412348 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -166,5 +166,5 @@ extra: latest_version: 0.6.0 snapshot_version: 0.6.0-SNAPSHOT tinkerpop_version: 3.4.10 - hadoop2_version: 2.7.7 + hadoop2_version: 2.8.5 jamm_version: 0.3.0 diff --git a/pom.xml b/pom.xml index f4393fb671..fc08cc66ae 100644 --- a/pom.xml +++ b/pom.xml @@ -78,10 +78,10 @@ 1.7.30 4.5.13 4.4.14 - 2.7.7 + 2.8.5 1.6.0 - 2.1.5 - ${hbase1.version} + 2.2.7 + 4.2.0-incubating 1.16.0 1.7.1 @@ -134,7 +134,7 @@ janusgraph-berkeleyje janusgraph-cql janusgraph-hadoop - janusgraph-hbase-parent + janusgraph-hbase janusgraph-bigtable janusgraph-es janusgraph-lucene