From 64e27525d3b0b4ad7abe129d822c8192a31ad1fe Mon Sep 17 00:00:00 2001 From: Frank Austin Nothaft Date: Fri, 14 Nov 2014 09:22:26 -0800 Subject: [PATCH] [ADAM-475] Removed code that has migrated to bdg-utils. --- .../adam/apis/java/JavaADAMContext.scala | 1 - .../org/bdgenomics/adam/cli/ADAMCommand.scala | 18 ++- adam-core/pom.xml | 12 +- .../org/bdgenomics/adam/io/ByteAccess.scala | 50 ------ .../adam/io/ByteArrayByteAccess.scala | 35 ----- .../bdgenomics/adam/io/ByteArrayLocator.scala | 24 --- .../org/bdgenomics/adam/io/FileLocator.scala | 52 ------- .../bdgenomics/adam/io/HTTPFileLocator.scala | 28 ---- .../adam/io/HTTPRangedByteAccess.scala | 146 ------------------ .../adam/io/LocalFileByteAccess.scala | 41 ----- .../bdgenomics/adam/io/LocalFileLocator.scala | 38 ----- .../org/bdgenomics/adam/io/S3ByteAccess.scala | 35 ----- .../bdgenomics/adam/io/S3FileLocator.scala | 34 ---- .../org/bdgenomics/adam/rdd/ADAMContext.scala | 1 - .../adam/util/CredentialsProperties.scala | 107 ------------- .../util/SerializableAWSCredentials.scala | 26 ---- .../org/bdgenomics/adam/util/TwoBitFile.scala | 3 +- .../bdgenomics/adam/io/ByteAccessSuite.scala | 124 --------------- .../bdgenomics/adam/io/FileLocatorSuite.scala | 87 ----------- .../adam/util/ClasspathFileLocator.scala | 41 ----- .../adam/util/CredentialPropertiesSuite.scala | 100 ------------ .../bdgenomics/adam/util/TwoBitSuite.scala | 2 +- pom.xml | 25 +-- 23 files changed, 19 insertions(+), 1011 deletions(-) delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/ByteAccess.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayByteAccess.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayLocator.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/FileLocator.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPFileLocator.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPRangedByteAccess.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileByteAccess.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileLocator.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/S3ByteAccess.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/io/S3FileLocator.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/util/CredentialsProperties.scala delete mode 100644 adam-core/src/main/scala/org/bdgenomics/adam/util/SerializableAWSCredentials.scala delete mode 100644 adam-core/src/test/scala/org/bdgenomics/adam/io/ByteAccessSuite.scala delete mode 100644 adam-core/src/test/scala/org/bdgenomics/adam/io/FileLocatorSuite.scala delete mode 100644 adam-core/src/test/scala/org/bdgenomics/adam/util/ClasspathFileLocator.scala delete mode 100644 adam-core/src/test/scala/org/bdgenomics/adam/util/CredentialPropertiesSuite.scala diff --git a/adam-apis/src/main/scala/org/bdgenomics/adam/apis/java/JavaADAMContext.scala b/adam-apis/src/main/scala/org/bdgenomics/adam/apis/java/JavaADAMContext.scala index 0cff0a5bc2..50d11d6975 100644 --- a/adam-apis/src/main/scala/org/bdgenomics/adam/apis/java/JavaADAMContext.scala +++ b/adam-apis/src/main/scala/org/bdgenomics/adam/apis/java/JavaADAMContext.scala @@ -19,7 +19,6 @@ package org.bdgenomics.adam.apis.java import org.apache.spark.SparkContext import org.apache.spark.api.java.JavaSparkContext -import org.bdgenomics.adam.instrumentation.ADAMMetricsListener import org.bdgenomics.adam.predicates.ADAMPredicate import org.bdgenomics.adam.rdd.ADAMContext import org.bdgenomics.adam.rdd.ADAMContext._ diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMCommand.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMCommand.scala index 82ff3f5e35..8df47cc5c7 100644 --- a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMCommand.scala +++ b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMCommand.scala @@ -17,13 +17,19 @@ */ package org.bdgenomics.adam.cli -import java.io.{ PrintStream, ByteArrayOutputStream } - +import java.io.{ + ByteArrayOutputStream, + PrintStream, + PrintWriter +} import org.apache.hadoop.mapreduce.Job import org.apache.spark.{ SparkConf, Logging, SparkContext } -import org.bdgenomics.adam.instrumentation.ADAMMetrics import org.bdgenomics.adam.util.HadoopUtil -import org.bdgenomics.utils.instrumentation.{ DurationFormatting, MetricsListener } +import org.bdgenomics.utils.instrumentation.{ + DurationFormatting, + MetricsListener, + RecordedMetrics +} trait ADAMCommandCompanion { val commandName: String @@ -48,7 +54,7 @@ trait ADAMSparkCommand[A <: Args4jBase] extends ADAMCommand with Logging { def run() { val start = System.nanoTime() - val metricsListener = if (args.printMetrics) Some(new MetricsListener(new ADAMMetrics())) else None + val metricsListener = if (args.printMetrics) Some(new MetricsListener(new RecordedMetrics())) else None val conf = new SparkConf().setAppName("adam: " + companion.commandName) if (conf.getOption("spark.master").isEmpty) { conf.setMaster("local[%d]".format(Runtime.getRuntime.availableProcessors())) @@ -70,7 +76,7 @@ trait ADAMSparkCommand[A <: Args4jBase] extends ADAMCommand with Logging { out.println() out.println("Overall Duration: " + DurationFormatting.formatNanosecondDuration(totalTime)) out.println() - listener.metrics.sparkMetrics.print(out) + listener.metrics.sparkMetrics.print(new PrintWriter(out)) logInfo("Metrics:" + bytes.toString("UTF-8")) }) } diff --git a/adam-core/pom.xml b/adam-core/pom.xml index f66004b0c7..cc1b398191 100644 --- a/adam-core/pom.xml +++ b/adam-core/pom.xml @@ -117,6 +117,10 @@ test-jar test + + org.bdgenomics.bdg-utils + bdg-utils-parquet + org.bdgenomics.bdg-utils bdg-utils-metrics @@ -182,14 +186,6 @@ scalatest_${scala.artifact.suffix} test - - com.amazonaws - aws-java-sdk - - - org.apache.httpcomponents - httpclient - diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteAccess.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteAccess.scala deleted file mode 100644 index 13b834105f..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteAccess.scala +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io._ - -/** - * ByteAccess is a wrapper trait around sources of bytes which are accessible at arbitrary offsets. - * - * (We can use this to wrap byte buffers, S3, FileInputStreams, or even range-accessible web services.) - */ -trait ByteAccess { - def length(): Long - def readByteStream(offset: Long, length: Int): InputStream - - /** - * readFully is a helper method, for when you're going to use readByteStream and just read - * all the bytes immediately. - * @param offset The offset in the resource at which you want to start reading - * @param length The number of bytes to read - * @return An Array of bytes read. The length of this array will be <= the 'length' argument. - */ - def readFully(offset: Long, length: Int): Array[Byte] = { - assert(length >= 0, "length %d should be non-negative".format(length)) - assert(offset >= 0, "offset %d should be non-negative".format(offset)) - var totalBytesRead: Int = 0 - val buffer = new Array[Byte](length) - val is = readByteStream(offset, length) - while (totalBytesRead < length) { - val bytesRead = is.read(buffer, totalBytesRead, length - totalBytesRead) - totalBytesRead += bytesRead - } - buffer - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayByteAccess.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayByteAccess.scala deleted file mode 100644 index a39e488c9b..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayByteAccess.scala +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io.{ ByteArrayInputStream, InputStream } - -class ByteArrayByteAccess(val bytes: Array[Byte]) extends ByteAccess with Serializable { - - private val inputStream = new ByteArrayInputStream(bytes) - assert(inputStream.markSupported(), "ByteArrayInputStream doesn't support marks") - - inputStream.mark(bytes.length) - - override def length(): Long = bytes.length - override def readByteStream(offset: Long, length: Int): InputStream = { - inputStream.reset() - inputStream.skip(offset) - inputStream - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayLocator.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayLocator.scala deleted file mode 100644 index a4b6bd44ca..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/ByteArrayLocator.scala +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -class ByteArrayLocator(val byteData: Array[Byte]) extends FileLocator { - override def relativeLocator(relativePath: String): FileLocator = this - override def parentLocator(): Option[FileLocator] = None - override def bytes: ByteAccess = new ByteArrayByteAccess(byteData) -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/FileLocator.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/FileLocator.scala deleted file mode 100644 index 2da8c581c7..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/FileLocator.scala +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -/** - * FileLocator is a trait which is meant to combine aspects of - * - Java's File - * - Hadoop's Path - * - S3 locations (including bucket and key) - * - classpath-relative URLs (classpath://, used in testing) - * - * It provides methods for relative addressing (parent and child locators, - * which are equivalent to the File(parent, child) constructor and the getParentFile method - * on the Java File class), as well as accessing the bytes named by a locator - * by retrieving a ByteAccess value. - * - * We're using implementations of FileLocator to provide a uniform access interface - * to Parquet files, whether they're in HDFS, a local filesystem, S3, or embedded in the - * classpath as part of tests. - */ -trait FileLocator extends Serializable { - - def parentLocator(): Option[FileLocator] - def relativeLocator(relativePath: String): FileLocator - def bytes: ByteAccess -} - -object FileLocator { - - val slashDivided = "^(.*)/([^/]+/?)$".r - - def parseSlash(path: String): Option[(String, String)] = - slashDivided.findFirstMatchIn(path) match { - case None => None - case Some(m) => Some(m.group(1), m.group(2)) - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPFileLocator.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPFileLocator.scala deleted file mode 100644 index 4b6c5117eb..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPFileLocator.scala +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.net.URI - -class HTTPFileLocator(uri: URI, retries: Int = 3) extends FileLocator { - override def parentLocator(): Option[FileLocator] = ??? - - override def relativeLocator(relativePath: String): FileLocator = ??? - - override def bytes: ByteAccess = new HTTPRangedByteAccess(uri, retries) -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPRangedByteAccess.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPRangedByteAccess.scala deleted file mode 100644 index 7a49e0737c..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/HTTPRangedByteAccess.scala +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io.InputStream -import java.net.URI - -import org.apache.http.client.HttpClient -import org.apache.http.client.methods.{ HttpGet, HttpHead } -import org.apache.http.impl.client.{ HttpClients, StandardHttpRequestRetryHandler } -import org.apache.spark.Logging - -/** - * HTTPRangedByteAccess supports Ranged GET queries against HTTP resources. - * - * @param uri The URL of the resource, which should support ranged queries - * @param retries Number of times to retry a failed connection before giving up - */ -class HTTPRangedByteAccess(uri: URI, retries: Int = 3) extends ByteAccess with Logging { - private def getClient: HttpClient = { - HttpClients.custom() - .setRetryHandler(new StandardHttpRequestRetryHandler(retries, true)) - .build() - } - - private def readLength(): Long = { - val head = new HttpHead(uri) - val response = getClient.execute(head) - val headers = response.getAllHeaders - - // The spec (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.5) says that - // Accept-Ranges can return values of 'bytes', 'none', or any other acceptable range-unit; - // however, "bytes" is the only range-unit recognized in HTTP/1.1 and that clients may choose - // to ignore other range-unit values. - // Therefore, we ignore non-standard range values, issuing only a warning when we see them. - // The spec says that _no_ Accept-Ranges header is possible and the resource may still - // respect range queries -- but if the Accept-Ranges header _is_ present, and the value is - // "none", then no range requests are allowed. In this case, that means we throw an - // exception. - headers.find(_.getName == "Accept-Ranges") match { - case Some(header) => header.getValue match { - case "none" => throw new IllegalStateException( - "Server for \"%s\" doesn't accept range requests (\"Accept-Ranges: none\" header in HEAD request response)" - .format(uri.toString)) - case "bytes" => - case value: String => logWarning( - "Server returned a header of \"Accept-Ranges: %s\", but the only values that we can handle are \"none\" and \"bytes\"".format(value)) - } - case None => - } - - // This is the Content-Length of the entire resource (as opposed to the Content-Length value - // on the range request, below -- that's just the length of the portion of the resource that - // was returned). - headers.find(_.getName == "Content-Length") match { - case None => throw new IllegalStateException("Unknown length of content for \"%s\" (No \"Content-Length\" header)" - .format(uri.toString)) - case Some(header) => header.getValue.toLong - } - } - - private lazy val _length = readLength() - - /* - Regex to read the Content-Range response header, as described in: - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16 - */ - private val byteRangeResponseRegex = "bytes\\s((?:\\*|\\d+-\\d+))/((?:\\*|\\d+))".r - - override def length(): Long = _length - - /** - * In this initial implementation, we throw an error when we get a partial response from the - * server whose content is less than the bytes we originally requested. - * - * @param offset The offset into the resource at which we want to start reading bytes - * @param length The number of bytes to be read - * @return An InputStream from a ranged HTTP GET, which should contain just the requested bytes - */ - override def readByteStream(offset: Long, length: Int): InputStream = { - val get = new HttpGet(uri) - - // I believe the 'end' coordinate on the range request is _inclusive_ - get.setHeader("Range", "bytes=%d-%d".format(offset, offset + length - 1)) - - val response = getClient.execute(get) - - // We want a 206 response to a ranged request, not your normal 200! - require(response.getStatusLine.getStatusCode == 206, - "Range request on \"%s\", expected status code 206 but received %d (%s)" - .format(uri.toString, response.getStatusLine.getStatusCode, response.getStatusLine.getReasonPhrase)) - - // This big nested set of case statements is to make sure that - // 1. we have a Content-Range in the header, - // 2. that the value of the Content-Range header matches the regex (see above), and - // 3. that the regex, if it contains a range field "start-end", that the - // a) 'start' value matches the 'offset' argument, and - // b) 'end' - 'start' + 1 matches the 'length' argument. - - // First, we check condition (1) - response.getAllHeaders.find(_.getName == "Content-Range") match { - case None => throw new IllegalStateException("Ranged GET didn't return a Content-Range header") - - // Now, we check condition (2) - case Some(header) => byteRangeResponseRegex.findFirstMatchIn(header.getValue) match { - case None => throw new IllegalStateException("Content-Range header value \"%s\" didn't match the expected format".format(header.getValue)) - - // Finally, we check condition (3) - case Some(m) => m.group(1) match { - case "*" => - case str: String => str.split("-").toSeq match { - case Seq(start, end) => - // Here is the check on (3a) - if (start.toLong != offset) - throw new IllegalArgumentException( - "Content-Range response start %d (from header \"%s\") doesn't match offset %d" - .format(start.toLong, header.getValue, offset)) - // here is the check on (3b) - if (end.toLong - start.toLong + 1 != length) - throw new IllegalArgumentException( - "Content-Range response length %d (from header \"%s\") doesn't match length %d" - .format(end.toLong - start.toLong + 1, header.getValue, length)) - } - - } - } - } - - response.getEntity.getContent - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileByteAccess.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileByteAccess.scala deleted file mode 100644 index 25ce1eb80e..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileByteAccess.scala +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io.{ File, FileInputStream, InputStream } - -/** - * This is somewhat poorly named, it probably should be LocalFileByteAccess - * - * @param f the file to read bytes from - */ -class LocalFileByteAccess(f: File) extends ByteAccess { - - assert(f.isFile, "\"%s\" isn't a file".format(f.getAbsolutePath)) - assert(f.exists(), "File \"%s\" doesn't exist".format(f.getAbsolutePath)) - assert(f.canRead, "File \"%s\" can't be read".format(f.getAbsolutePath)) - - override def length(): Long = f.length() - - override def readByteStream(offset: Long, length: Int): InputStream = { - val fileIo = new FileInputStream(f) - fileIo.skip(offset) - fileIo - } - -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileLocator.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileLocator.scala deleted file mode 100644 index a574b0ffe0..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/LocalFileLocator.scala +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io.File - -class LocalFileLocator(val file: File) extends FileLocator { - override def relativeLocator(relativePath: String): FileLocator = new LocalFileLocator(new File(file, relativePath)) - override def bytes: ByteAccess = new LocalFileByteAccess(file) - - override def parentLocator(): Option[FileLocator] = file.getParentFile match { - case null => None - case parentFile: File => Some(new LocalFileLocator(parentFile)) - } - - override def hashCode(): Int = file.hashCode() - override def equals(x: Any): Boolean = { - x match { - case loc: LocalFileLocator => file.equals(loc.file) - case _ => false - } - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/S3ByteAccess.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/S3ByteAccess.scala deleted file mode 100644 index 3419ef48fb..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/S3ByteAccess.scala +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.GetObjectRequest -import java.io.InputStream - -class S3ByteAccess(client: AmazonS3, bucket: String, keyName: String) extends ByteAccess { - assert(bucket != null) - assert(keyName != null) - - lazy val objectMetadata = client.getObjectMetadata(bucket, keyName) - override def length(): Long = objectMetadata.getContentLength - override def readByteStream(offset: Long, length: Int): InputStream = { - val getObjectRequest = new GetObjectRequest(bucket, keyName).withRange(offset, offset + length) - client.getObject(getObjectRequest).getObjectContent - } - -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/io/S3FileLocator.scala b/adam-core/src/main/scala/org/bdgenomics/adam/io/S3FileLocator.scala deleted file mode 100644 index e41e3c6588..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/io/S3FileLocator.scala +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import com.amazonaws.auth.AWSCredentials -import com.amazonaws.services.s3.AmazonS3Client - -class S3FileLocator(val credentials: AWSCredentials, val bucket: String, val key: String) extends FileLocator { - - override def parentLocator(): Option[FileLocator] = FileLocator.parseSlash(key) match { - case Some((parent, child)) => Some(new S3FileLocator(credentials, bucket, parent)) - case None => None - } - - override def relativeLocator(relativePath: String): FileLocator = - new S3FileLocator(credentials, bucket, "%s/%s".format(key.stripSuffix("/"), relativePath)) - - override def bytes: ByteAccess = new S3ByteAccess(new AmazonS3Client(credentials), bucket, key) -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/ADAMContext.scala b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/ADAMContext.scala index 168a22c79e..6e8df63d73 100644 --- a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/ADAMContext.scala +++ b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/ADAMContext.scala @@ -29,7 +29,6 @@ import org.apache.spark.rdd.RDD import org.apache.spark.scheduler.StatsReportListener import org.apache.spark.{ Logging, SparkConf, SparkContext } import org.bdgenomics.adam.converters.SAMRecordConverter -import org.bdgenomics.adam.instrumentation.ADAMMetricsListener import org.bdgenomics.adam.models._ import org.bdgenomics.adam.predicates.ADAMPredicate import org.bdgenomics.adam.projections.{ AlignmentRecordField, NucleotideContigFragmentField, Projection } diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/util/CredentialsProperties.scala b/adam-core/src/main/scala/org/bdgenomics/adam/util/CredentialsProperties.scala deleted file mode 100644 index 3248a30bf6..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/util/CredentialsProperties.scala +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.util - -import java.io.{ FileInputStream, File } -import org.apache.spark.Logging -import scala.io.Source -import com.amazonaws.auth.AWSCredentials - -/** - * CredentialsProperties is a wrapper class which extracts Amazon S3 keys (although it could be - * modified to extract other key / secret-key pairs) from a key-value-formatted file (see below), - * if available, or from the System environment otherwise. - * - * (This is to make testing and running on a local environment easier.) - * - * The 'location' parameter names a file which should have the format of - * key=value - * one per line. Whitespace around both 'key' and 'value' are stripped. - * - * @param location The location of the file which will be read _if it exists and is readable_, - * otherwise the parameters will be read from the System environment. - */ -class CredentialsProperties(location: Option[File]) extends Serializable with Logging { - - val defaultAccessKey = System.getenv("AWS_ACCESS_KEY_ID") - val defaultSecretKey = System.getenv("AWS_SECRET_KEY") - private val defaultMap = Map("accessKey" -> defaultAccessKey, "secretKey" -> defaultSecretKey) - val configuration = location.map(new ConfigurationFile(_, Some(defaultMap))).map(_.properties).getOrElse(defaultMap) - - /** - * Retrieves the value associated with a given key from the configuration file. - * - * The optional 'suffix' argument is used for differentiating different key/value pairs, - * used for different purposes but with similar keys. For example, we (Genome Bridge) - * will create configuration files with one key 'accessKey' which contains the AWS access key - * for accessing almost all our AWS resources -- however, we use a different access (and secret) - * key for accessing S3, and that's put into the same configuration file with a key name - * 'accessKey_s3'. (Similarly for 'secretKey' and 'secretKey_s3'). - * - * @param keyName The base key used to retrieve a value - * @param suffix If suffix is None, then the base key is used to retrieve the corresponding - * value. If suffix is specified, then '[base key]_[suffix]' is used instead. - * If '[base key]_[suffix]' doesn't exist in the file, then the properties - * falls back to retrieving the value just associated with the base key itself. - * @return The (string) value associated with the given key, or null if no such key exists. - */ - def configuredValue(keyName: String, suffix: Option[String] = None): String = { - suffix match { - case None => configuration(keyName) - case Some(suffixString) => - val combinedKey = "%s_%s".format(keyName, suffixString) - if (configuration.contains(combinedKey)) { - configuration(combinedKey) - } else { - configuration(keyName) - } - } - } - - def accessKey(suffix: Option[String]): String = configuredValue("accessKey", suffix) - def secretKey(suffix: Option[String]): String = configuredValue("secretKey", suffix) - - def awsCredentials(suffix: Option[String] = None): AWSCredentials = { - new SerializableAWSCredentials(accessKey(suffix), secretKey(suffix)) - } -} - -private[util] case class ConfigurationFile(properties: Map[String, String]) extends Serializable { - def this(f: File, defaultValues: Option[Map[String, String]] = None) = this(ConfigurationParser(f, defaultValues)) -} - -private[util] object ConfigurationParser extends Logging { - - def apply(f: File, defaultValues: Option[Map[String, String]] = None): Map[String, String] = { - if (!f.exists() || !f.canRead) { - logWarning("File \"%s\" does not exist, using default values.".format(f.getAbsolutePath)) - defaultValues.get - - } else { - logInfo("Reading configuration values from \"%s\"".format(f.getAbsolutePath)) - val is = new FileInputStream(f) - // we wrote our own key=value parsing, since we were having problems with whitespace - // using the Java util.Properties parsing. - val lines = Source.fromInputStream(is).getLines().map(_.trim) - val nonComments = lines.filter(line => !line.startsWith("#") && line.contains("=")) - val map = nonComments.map(_.split("=")).map(array => array.map(_.trim)).map(array => (array(0), array(1))).toMap - is.close() - map - } - } -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/util/SerializableAWSCredentials.scala b/adam-core/src/main/scala/org/bdgenomics/adam/util/SerializableAWSCredentials.scala deleted file mode 100644 index 7d8d50298d..0000000000 --- a/adam-core/src/main/scala/org/bdgenomics/adam/util/SerializableAWSCredentials.scala +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.util - -import com.amazonaws.auth.AWSCredentials - -class SerializableAWSCredentials(accessKey: String, secretKey: String) extends AWSCredentials with Serializable { - def this() = this(System.getenv("AWS_ACCESS_KEY_ID"), System.getenv("AWS_SECRET_KEY")) - def getAWSAccessKeyId: String = accessKey - def getAWSSecretKey: String = secretKey -} diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/util/TwoBitFile.scala b/adam-core/src/main/scala/org/bdgenomics/adam/util/TwoBitFile.scala index aab6288355..edf6eb915e 100644 --- a/adam-core/src/main/scala/org/bdgenomics/adam/util/TwoBitFile.scala +++ b/adam-core/src/main/scala/org/bdgenomics/adam/util/TwoBitFile.scala @@ -19,8 +19,7 @@ package org.bdgenomics.adam.util import java.nio.{ ByteOrder, ByteBuffer } - -import org.bdgenomics.adam.io.ByteAccess +import org.bdgenomics.utils.parquet.io.ByteAccess import org.bdgenomics.adam.models.ReferenceRegion object TwoBitFile { diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/io/ByteAccessSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/io/ByteAccessSuite.scala deleted file mode 100644 index 022bb51615..0000000000 --- a/adam-core/src/test/scala/org/bdgenomics/adam/io/ByteAccessSuite.scala +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import com.amazonaws.services.s3.AmazonS3Client -import java.io.{ File, PrintWriter } -import java.net.{ ServerSocket, URI } -import java.util.concurrent.atomic.AtomicInteger -import org.bdgenomics.adam.util.CredentialsProperties -import org.bdgenomics.utils.misc.{ NetworkConnected, S3Test } -import org.scalatest.FunSuite - -class ByteAccessSuite extends FunSuite { - - lazy val credentials = new CredentialsProperties(Some(new File(System.getProperty("user.home") + "/spark.conf"))) - .awsCredentials(Some("s3")) - - lazy val bucketName = System.getenv("BUCKET_NAME") - lazy val parquetLocation = System.getenv("PARQUET_LOCATION") - - test("ByteArrayByteAccess returns arbitrary subsets of bytes correctly") { - val bytes = Array[Byte](0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - val access = new ByteArrayByteAccess(bytes) - - assert(access.length() === bytes.length) - assert(access.readFully(5, 5) === bytes.slice(5, 10)) - } - - test("ByteArrayByteAccess supports two successive calls with different offsets") { - val bytes = Array[Byte](0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - val access = new ByteArrayByteAccess(bytes) - - assert(access.length() === bytes.length) - assert(access.readFully(5, 5) === bytes.slice(5, 10)) - assert(access.readFully(3, 5) === bytes.slice(3, 8)) - } - - test("LocalFileByteAccess returns arbitrary subsets of bytes correctly") { - val content = "abcdefghij" - val temp = File.createTempFile("byteaccesssuite", "test") - - val writer = new PrintWriter(temp) - writer.print(content) - writer.close() - - val access = new LocalFileByteAccess(temp) - assert(access.length() === content.length()) - assert(access.readFully(3, 5) === content.substring(3, 8).getBytes("ASCII")) - } - - test("HTTPRangedByteAccess will retry multiple times", NetworkConnected) { - val socket = new ServerSocket(0) - if (!socket.isBound) throw new Exception("Could not bind ServerSocket") - val port = socket.getLocalPort - val count = new AtomicInteger(0) - val thread = new Thread(new Runnable { - def single() = { - val client = socket.accept() - count.getAndIncrement - client.close() - } - - override def run(): Unit = { - single() - run() - } - }) - - thread.start() - - val uri = URI.create("http://localhost:" + port + "/") - val http = new HTTPRangedByteAccess(uri, 9) - intercept[Exception] { - http.length() - } - assert(count.get() === 10, "Retrying 9 times should result in 10 connections") - socket.close() - thread.interrupt() - } - - test("HTTPRangedByteAccess supports range queries", NetworkConnected) { - val uri = URI.create("https://s3.amazonaws.com/bdgenomics-test/mouse_chrM.bam") - val http = new HTTPRangedByteAccess(uri, 1) - val bytes1 = http.readFully(100, 10) - val bytes2 = http.readFully(100, 100) - - assert(bytes1.length === 10) - assert(bytes2.length === 100) - assert(bytes1 === bytes2.slice(0, 10)) - - // figured this out by executing: - // curl --range 100-109 http://www.cs.berkeley.edu/~massie/bams/mouse_chrM.bam | od -t u1 - assert(bytes1 === Array(188, 185, 119, 110, 102, 222, 76, 23, 189, 139).map(_.toByte)) - } - - test("HTTPRangedByteAccess can retrieve a full range", NetworkConnected) { - val uri = URI.create("https://s3.amazonaws.com/bdgenomics-test/eecslogo.gif") - val http = new HTTPRangedByteAccess(uri, 1) - val bytes = http.readFully(0, http.length().toInt) - assert(bytes.length === http.length()) - } - - test("Testing S3 byte access", NetworkConnected, S3Test) { - val byteAccess = new S3ByteAccess(new AmazonS3Client(credentials), - bucketName, - parquetLocation) - assert(byteAccess.readFully(0, 1)(0) === 80) - } -} diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/io/FileLocatorSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/io/FileLocatorSuite.scala deleted file mode 100644 index d2fa8ccc28..0000000000 --- a/adam-core/src/test/scala/org/bdgenomics/adam/io/FileLocatorSuite.scala +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.io - -import java.io.{ File, FileWriter, PrintWriter } - -import org.scalatest.FunSuite - -class FileLocatorSuite extends FunSuite { - - test("parseSlash can correctly parse a one-slash string") { - assert(FileLocator.parseSlash("foo/bar") === Some(("foo", "bar"))) - assert(FileLocator.parseSlash("/foo") === Some(("", "foo"))) - } - - test("parseSlash can correctly parse a two-slash string") { - assert(FileLocator.parseSlash("foo/bar") === Some(("foo", "bar"))) - assert(FileLocator.parseSlash("/foo/bar") === Some(("/foo", "bar"))) - } - - test("parseSlash can correctly parse a no-slash string") { - assert(FileLocator.parseSlash("foo") === None) - } -} - -class LocalFileLocatorSuite extends FunSuite { - - test("parentLocator retrieves the parent directory") { - val temp: File = File.createTempFile("LocalFileLocatorSuite", "test") - val loc: FileLocator = new LocalFileLocator(temp) - val parentLocOpt: Option[FileLocator] = loc.parentLocator() - - parentLocOpt match { - case Some(parentLoc) => assert(parentLoc === new LocalFileLocator(temp.getParentFile)) - case None => fail("parentLoc wasn't defined") - } - } - - test("relativeLocator retrieves a subdirectory") { - val temp1: File = File.createTempFile("LocalFileLocatorSuite", "test") - val tempDir = temp1.getParentFile - - val tempDirLoc = new LocalFileLocator(tempDir) - val tempLoc = tempDirLoc.relativeLocator(temp1.getName) - - assert(tempLoc === new LocalFileLocator(temp1)) - } - - test("bytes accesses the named underlying file") { - val temp = File.createTempFile("LocalFileLocatorSuite", "test") - val pw = new PrintWriter(new FileWriter(temp)) - pw.println("abcdefghij") - pw.close() - - val loc = new LocalFileLocator(temp) - val io = loc.bytes - - val str = new String(io.readFully(3, 3), "UTF-8") - assert(str === "def") - } -} - -class ByteArrayLocatorSuite extends FunSuite { - test("byte access can be wrapped in a locator correctly") { - val bytes = Array[Byte](1, 2, 3, 4, 5) - val loc = new ByteArrayLocator(bytes) - - val io = loc.bytes - val read = io.readFully(2, 3) - assert(read === Array[Byte](3, 4, 5)) - } -} diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/util/ClasspathFileLocator.scala b/adam-core/src/test/scala/org/bdgenomics/adam/util/ClasspathFileLocator.scala deleted file mode 100644 index c00d431007..0000000000 --- a/adam-core/src/test/scala/org/bdgenomics/adam/util/ClasspathFileLocator.scala +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.services - -import java.io.File - -import org.bdgenomics.adam.io.{ ByteAccess, FileLocator, LocalFileByteAccess } - -class ClasspathFileLocator(classpath: String) extends FileLocator { - override def relativeLocator(relativePath: String): FileLocator = - new ClasspathFileLocator("%s/%s".format(classpath.stripSuffix("/"), relativePath)) - - override def bytes: ByteAccess = { - val url = Thread.currentThread().getContextClassLoader.getResource(classpath) - if (url == null) { throw new IllegalArgumentException("Illegal classpath \"%s\"".format(classpath)) } - val path = url.getFile - val file = new File(path) - println("Returning bytes from %s".format(file.getAbsolutePath)) - new LocalFileByteAccess(file) - } - - override def parentLocator(): Option[FileLocator] = FileLocator.parseSlash(classpath) match { - case Some((parent, child)) => Some(new ClasspathFileLocator(parent)) - case None => None - } -} diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/util/CredentialPropertiesSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/util/CredentialPropertiesSuite.scala deleted file mode 100644 index fb6bfe3066..0000000000 --- a/adam-core/src/test/scala/org/bdgenomics/adam/util/CredentialPropertiesSuite.scala +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to Big Data Genomics (BDG) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The BDG licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.bdgenomics.adam.util - -import org.scalatest.FunSuite -import java.io.File - -class CredentialsPropertiesTestSuite extends FunSuite { - - test("Can parse a simple configuration file with CredentialsProperties") { - val path = Thread.currentThread().getContextClassLoader.getResource("test.conf").getFile - val file = new File(path) - val cp = new CredentialsProperties(Some(file)) - - val aws = cp.awsCredentials() - assert(aws.getAWSAccessKeyId === "accessKey") - assert(aws.getAWSSecretKey === "secretKey") - - val aws_s3 = cp.awsCredentials(Some("s3")) - assert(aws_s3.getAWSAccessKeyId === "accessKey_s3") - assert(aws_s3.getAWSSecretKey === "secretKey_s3") - - } - -} - -class ConfigurationFileSuite extends FunSuite { - - test("Can read values out of a file") { - val path = Thread.currentThread().getContextClassLoader.getResource("test.conf").getFile - val file = new File(path) - val config = new ConfigurationFile(file) - assert(config.properties.contains("accessKey")) - assert(config.properties.contains("secretKey")) - assert(config.properties.contains("accessKey_s3")) - assert(config.properties.contains("secretKey_s3")) - assert(config.properties("accessKey") === "accessKey") - assert(config.properties("secretKey") === "secretKey") - assert(config.properties("accessKey_s3") === "accessKey_s3") - assert(config.properties("secretKey_s3") === "secretKey_s3") - } - - test("Reads default values when the file does not exist.") { - val path = "/foo/bar.conf" - val file = new File(path) - val defaultMap = Seq("accessKey" -> "foo", "secretKey" -> "bar").toMap - val config = new ConfigurationFile(file, Some(defaultMap)) - assert(config.properties.contains("accessKey")) - assert(config.properties.contains("secretKey")) - assert(config.properties("accessKey") === "foo") - assert(config.properties("secretKey") === "bar") - } - - test("Does not read the default values, when the file does exist.") { - val path = Thread.currentThread().getContextClassLoader.getResource("test.conf").getFile - val file = new File(path) - val defaultMap = Seq("accessKey" -> "foo", "secretKey" -> "bar").toMap - val config = new ConfigurationFile(file, Some(defaultMap)) - assert(config.properties.contains("accessKey")) - assert(config.properties.contains("secretKey")) - assert(config.properties("accessKey") === "accessKey") - assert(config.properties("secretKey") === "secretKey") - } - -} - -class ConfigurationParserSuite extends FunSuite { - test("parses a simple file") { - val path = Thread.currentThread().getContextClassLoader.getResource("test.conf").getFile - val file = new File(path) - val map: Map[String, String] = ConfigurationParser(file) - - assert(map.size === 4) - - assert(map.contains("accessKey")) - assert(map.contains("secretKey")) - assert(map("accessKey") === "accessKey") - assert(map("secretKey") === "secretKey") - - assert(map.contains("accessKey_s3")) - assert(map.contains("secretKey_s3")) - assert(map("accessKey_s3") === "accessKey_s3") - assert(map("secretKey_s3") === "secretKey_s3") - } -} diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/util/TwoBitSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/util/TwoBitSuite.scala index 3716ad8c41..3673e6946f 100644 --- a/adam-core/src/test/scala/org/bdgenomics/adam/util/TwoBitSuite.scala +++ b/adam-core/src/test/scala/org/bdgenomics/adam/util/TwoBitSuite.scala @@ -19,7 +19,7 @@ package org.bdgenomics.adam.util import java.io.File -import org.bdgenomics.adam.io.LocalFileByteAccess +import org.bdgenomics.utils.parquet.io.LocalFileByteAccess import org.bdgenomics.adam.models.ReferenceRegion import org.scalatest.FunSuite diff --git a/pom.xml b/pom.xml index da980c70eb..029f2c17b4 100644 --- a/pom.xml +++ b/pom.xml @@ -23,8 +23,7 @@ 1.7.6 1.2.0 1.6.0rc4 - 0.0.1 - 0.0.1-SNAPSHOT + 0.0.2-SNAPSHOT 2.2.0 0.99.2 @@ -454,28 +453,6 @@ slf4j-log4j12 1.7.5 - - com.amazonaws - aws-java-sdk - 1.7.5 - provided - - - org.apache.httpcomponents - httpclient - 4.3.2 - - - com.netflix.servo - servo-core - 0.5.5 - - - com.google.guava - guava - - - org.fusesource.scalate scalate-core_2.10