diff --git a/.gitignore b/.gitignore index 1e21302f..c4498bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,6 @@ server.pid /dist/ .cache +### Redis ### +dump.rdb + diff --git a/build.sbt b/build.sbt index 85711636..8fac8462 100755 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,15 @@ lazy val commonSettings = Seq( version := "0.12.1-SNAPSHOT", scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation", "-feature", "-Xlint"), javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map { case (key, value) => "-D" + key + "=" + value }.toSeq, - testOptions in Test += Tests.Argument("-oDF"), + testOptions in Test ++= Seq( + Tests.Argument("-oDF"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "CommonTest"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V1Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V2Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V3Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V4Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "HBaseTest") + ), parallelExecution in Test := false, resolvers ++= Seq( Resolver.mavenLocal, @@ -49,7 +57,23 @@ lazy val s2rest_netty = project .dependsOn(s2core) .settings(commonSettings: _*) -lazy val s2core = project.settings(commonSettings: _*) +lazy val HBaseTest = config("hbase") extend(Test) +lazy val RedisTest = config("redis") extend(Test) + +lazy val s2core = project.settings(commonSettings: _*). + configs(HBaseTest). + configs(RedisTest). + settings(inConfig(RedisTest)(Defaults.testTasks): _*). + settings( + testOptions in RedisTest --= Seq( + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V1Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V2Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V3Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "HBaseTest") + ), + testOptions in RedisTest += Tests.Argument(TestFrameworks.ScalaTest, "-n", "RedisTest") + ) + lazy val spark = project.settings(commonSettings: _*) diff --git a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala index 0937f4c7..10d9245a 100644 --- a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala +++ b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala @@ -20,7 +20,7 @@ package org.apache.s2graph.loader.subscriber import org.apache.s2graph.core.Management -import org.apache.s2graph.core.types.HBaseType +import org.apache.s2graph.core.types.GraphType import org.apache.spark.{SparkConf, SparkContext} import org.scalatest._ import TransferToHFile._ @@ -81,7 +81,7 @@ class TransferToHFileTest extends FlatSpec with BeforeAndAfterAll with Matchers Seq(), "weak", None, None, - HBaseType.DEFAULT_VERSION, + GraphType.DEFAULT_VERSION, false, Management.defaultCompressionAlgorithm ) diff --git a/s2core/build.sbt b/s2core/build.sbt index 6774c60c..b99a88b0 100644 --- a/s2core/build.sbt +++ b/s2core/build.sbt @@ -35,7 +35,8 @@ libraryDependencies ++= Seq( "org.scalikejdbc" %% "scalikejdbc" % "2.1.+", "mysql" % "mysql-connector-java" % "5.1.28", "org.apache.kafka" % "kafka-clients" % "0.8.2.0" excludeAll(ExclusionRule(organization = "org.slf4j"), ExclusionRule(organization = "com.sun.jdmk"), ExclusionRule(organization = "com.sun.jmx"), ExclusionRule(organization = "javax.jms")), - "com.github.danielwegener" % "logback-kafka-appender" % "0.0.4" + "com.github.danielwegener" % "logback-kafka-appender" % "0.0.4", + "redis.clients" % "jedis" % "2.7.0" ) libraryDependencies := { diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index 1840306a..34162125 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -45,15 +45,20 @@ cache.ttl.seconds=60 cache.max.size=100000 # DB -s2graph.models.table.name = "models-dev" -db.default.driver = "com.mysql.jdbc.Driver" -db.default.url = "jdbc:mysql://"${host}":3306/graph_dev" -db.default.user = "graph" -db.default.password = "graph" +s2graph.models.table.name="models-dev" +db.default.driver="com.mysql.jdbc.Driver" +db.default.url="jdbc:mysql://"${host}":3306/graph_dev" +db.default.user="graph" +db.default.password="graph" akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] - loglevel = "DEBUG" + loggers=["akka.event.slf4j.Slf4jLogger"] + loglevel="DEBUG" } +# hbase, redis, etc. +s2graph.storage.backend=redis + +# for redis +# redis.storage=["localhost:6379"] diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala index ac1d4c12..05d515d0 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala @@ -22,6 +22,7 @@ package org.apache.s2graph.core import java.util import java.util.concurrent.ConcurrentHashMap +import com.kakao.s2graph.core.storage.redis.RedisStorage import com.typesafe.config.{Config, ConfigFactory} import org.apache.s2graph.core.mysqls.{Label, Model} import org.apache.s2graph.core.parsers.WhereParser @@ -59,8 +60,7 @@ object Graph { "delete.all.fetch.size" -> java.lang.Integer.valueOf(1000), "future.cache.max.size" -> java.lang.Integer.valueOf(100000), "future.cache.expire.after.write" -> java.lang.Integer.valueOf(10000), - "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000), - "s2graph.storage.backend" -> "hbase" + "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000) ) var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs) @@ -351,6 +351,7 @@ object Graph { def initStorage(config: Config)(ec: ExecutionContext) = { config.getString("s2graph.storage.backend") match { case "hbase" => new AsynchbaseStorage(config)(ec) + case "redis" => new RedisStorage(config)(ec) case _ => throw new RuntimeException("not supported storage.") } } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala index 0898ffa4..e917859f 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala @@ -44,4 +44,6 @@ object GraphExceptions { case class FetchTimeoutException(msg: String) extends Exception(msg) case class DropRequestException(msg: String) extends Exception(msg) + + case class UnsupportedVersionException(msg: String) extends Exception(msg) } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala index ebfee7a4..4d0e318c 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala @@ -155,4 +155,8 @@ object GraphUtil { } } + def bytesToHexString(b: Array[Byte]): String = { + val tmp = b.map("%02x".format(_)).mkString("\\x") + if ( tmp.isEmpty ) "" else "\\x" + tmp + } } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala index f9b74314..aff4587e 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala @@ -22,7 +22,7 @@ package org.apache.s2graph.core import org.apache.s2graph.core.GraphExceptions.{InvalidHTableException, LabelAlreadyExistException, LabelNotExistException} import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} import org.apache.s2graph.core.mysqls._ -import org.apache.s2graph.core.types.HBaseType._ +import org.apache.s2graph.core.types.GraphType._ import org.apache.s2graph.core.types._ import play.api.libs.json.Reads._ import play.api.libs.json._ @@ -45,7 +45,7 @@ object Management extends JSONParser { } - import HBaseType._ + import GraphType._ val DefaultCompressionAlgorithm = "gz" diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala index 6e17793a..041cd14b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala @@ -22,7 +22,7 @@ package org.apache.s2graph.core.mysqls import org.apache.s2graph.core.GraphExceptions.ModelNotFoundException import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.core.{GraphExceptions, GraphUtil, JSONParser} +import org.apache.s2graph.core.{GraphUtil, JSONParser} import play.api.libs.json.Json import scalikejdbc._ diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala index afda6f95..e3baa622 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala @@ -548,7 +548,7 @@ class RequestParser(config: Config) extends JSONParser { // expect new label don`t provide hTableName val hTableName = (jsValue \ "hTableName").asOpt[String] val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int] - val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(HBaseType.DEFAULT_VERSION) + val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(GraphType.DEFAULT_VERSION) val isAsync = (jsValue \ "isAsync").asOpt[Boolean].getOrElse(false) val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm) diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala index 55b3e79f..164115f1 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala @@ -217,7 +217,7 @@ class RestHandler(graph: Graph)(implicit ec: ExecutionContext) { } } - private def getVertices(jsValue: JsValue) = { + def getVertices(jsValue: JsValue) = { val jsonQuery = jsValue val ts = System.currentTimeMillis() val props = "{}" diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala index d82c5076..ddc8463f 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala @@ -35,7 +35,7 @@ trait Deserializable[E] extends StorageDeserializable[E] { pos += srcIdLen val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) pos += 4 - val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsSnapshot(kv.row, pos) val rowLen = srcIdLen + 4 + 1 (srcVertexId, labelWithDir, labelIdxSeq, isInverted, rowLen) diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala index b690307b..df4b3b01 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala @@ -27,6 +27,7 @@ object SKeyValue { val Put = 1 val Delete = 2 val Increment = 3 + val SnapshotPut = 4 // redis exclusive val Default = Put } case class SKeyValue(table: Array[Byte], diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala index fd968add..cb6c3c6a 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala @@ -39,7 +39,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Random, Try} abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { - import HBaseType._ + import GraphType._ /** storage dependent configurations */ val MaxRetryNum = config.getInt("max.retry.number") @@ -94,7 +94,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { * @param vertex: vertex to serialize * @return serializer implementation */ - def vertexSerializer(vertex: Vertex) = new VertexSerializable(vertex) + def vertexSerializer(vertex: Vertex): Serializable[Vertex] = new VertexSerializable(vertex) /** * create deserializer that can parse stored CanSKeyValue into snapshotEdge. @@ -259,9 +259,6 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { compressionAlgorithm: String): Unit - - - /** Public Interface */ def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = { @@ -276,6 +273,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { val queryParam = QueryParam.Empty val q = Query.toQuery(Seq(vertex), queryParam) val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam) + fetchVertexKeyValues(buildRequest(queryRequest)).map { kvs => fromResult(queryParam, kvs, vertex.serviceColumn.schemaVersion) } recoverWith { case ex: Throwable => @@ -316,7 +314,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { val weakEdgesFutures = weakEdges.groupBy { e => e.label.hbaseZkAddr }.map { case (zkQuorum, edges) => val mutations = edges.flatMap { edge => val (_, edgeUpdate) = - if (edge.op == GraphUtil.operations("delete")) Edge.buildDeleteBulk(None, edge) + if (edge.op == GraphUtil.operations("delete")) {logger.debug(s">>>>> op == delete"); Edge.buildDeleteBulk(None, edge)} else Edge.buildOperation(None, Seq(edge)) buildVertexPutsAsync(edge) ++ indexedEdgeMutations(edgeUpdate) ++ snapshotEdgeMutations(edgeUpdate) ++ increments(edgeUpdate) @@ -442,6 +440,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { } future recoverWith { case FetchTimeoutException(retryEdge) => + logger.error(s"\n[[ fetch timeout exception") logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}") retry(tryNum + 1)(edges, statusCode)(fn) @@ -456,6 +455,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { Thread.sleep(Random.nextInt(MaxBackOff)) logger.info(s"[Try: $tryNum], [Status: $status] partial fail.\n${retryEdge.toLogString}\nFailReason: ${faileReason}") + logger.error(s"\n[[ Try : $tryNum - $status") retry(tryNum + 1)(Seq(retryEdge), failedStatusCode)(fn) case ex: Exception => logger.error("Unknown exception", ex) @@ -545,7 +545,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { } yield { val label = queryRequest.queryParam.label label.schemaVersion match { - case HBaseType.VERSION3 | HBaseType.VERSION4 => + case GraphType.VERSION3 | GraphType.VERSION4 => if (label.consistencyLevel == "strong") { /** * read: snapshotEdge on queryResult = O(N) @@ -816,7 +816,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { logger.debug(log) // debug(ret, "acquireLock", edge.toSnapshotEdge) } else { - throw new PartialFailureException(edge, 0, "hbase fail.") + throw new PartialFailureException(edge, 0, "acquireLock failed") } true } @@ -837,6 +837,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { } val p = Random.nextDouble() if (p < FailProb) throw new PartialFailureException(edge, 3, s"$p") +// if (p < 0.3) throw new PartialFailureException(edge, 3, "aaa releaseLock fail.") else { val releaseLockEdgePut = snapshotEdgeSerializer(releaseLockEdge).toKeyValues.head val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head @@ -848,6 +849,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { if (ret) { debug(ret, "releaseLock", edge.toSnapshotEdge) } else { + logger.error(s"\n[[ release lock failed") val msg = Seq("\nFATAL ERROR\n", "=" * 50, oldBytes.toList, @@ -860,7 +862,7 @@ abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) { ) logger.error(msg.mkString("\n")) // error(ret, "releaseLock", edge.toSnapshotEdge) - throw new PartialFailureException(edge, 3, "hbase fail.") + throw new PartialFailureException(edge, 3, "aaa releaseLock fail.") } true } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala index 69926fa3..f2315e99 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala @@ -21,12 +21,12 @@ package org.apache.s2graph.core.storage import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.QueryParam -import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, InnerValLikeWithTs} +import org.apache.s2graph.core.types.{GraphType, InnerVal, InnerValLike, InnerValLikeWithTs} import org.apache.s2graph.core.utils.logger object StorageDeserializable { /** Deserializer */ - def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = { + def bytesToLabelIndexSeqWithIsSnapshot(bytes: Array[Byte], offset: Int): (Byte, Boolean) = { val byte = bytes(offset) val isInverted = if ((byte & 1) != 0) true else false val labelOrderSeq = byte >> 1 @@ -85,7 +85,7 @@ object StorageDeserializable { val kvs = new Array[(Byte, InnerValLike)](len) var i = 0 while (i < len) { - val k = HBaseType.EMPTY_SEQ_BYTE + val k = GraphType.EMPTY_SEQ_BYTE val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version) pos += numOfBytesUsed kvs(i) = (k -> v) diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala index b6435e46..098df0de 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala @@ -48,9 +48,9 @@ object StorageSerializable { bytes } - def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = { + def labelOrderSeqWithIsSnapshot(labelOrderSeq: Byte, isSnapshot: Boolean): Array[Byte] = { assert(labelOrderSeq < (1 << 6)) - val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0) + val byte = labelOrderSeq << 1 | (if (isSnapshot) 1 else 0) Array.fill(1)(byte.toByte) } } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala index 66a1be48..43240f87 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala @@ -35,7 +35,7 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.s2graph.core._ import org.apache.s2graph.core.mysqls.LabelMeta import org.apache.s2graph.core.storage._ -import org.apache.s2graph.core.types.{HBaseType, VertexId} +import org.apache.s2graph.core.types.{GraphType, VertexId} import org.apache.s2graph.core.utils.{DeferCache, Extensions, FutureCache, logger} import org.hbase.async._ @@ -196,7 +196,7 @@ class AsynchbaseStorage(override val config: Config)(implicit ec: ExecutionConte val (minTs, maxTs) = queryParam.duration.getOrElse((0L, Long.MaxValue)) label.schemaVersion match { - case HBaseType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty => + case GraphType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty => val scanner = client.newScanner(label.hbaseTableName.getBytes) scanner.setFamily(edgeCf) @@ -208,7 +208,7 @@ class AsynchbaseStorage(override val config: Config)(implicit ec: ExecutionConte val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes val labelWithDirBytes = indexEdge.labelWithDir.bytes - val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false) + val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsSnapshot(indexEdge.labelIndexSeq, isSnapshot = false) // val labelIndexSeqWithIsInvertedStopBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = true) val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, Bytes.add(labelIndexSeqWithIsInvertedBytes, Array.fill(1)(edge.op))) val (startKey, stopKey) = diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisGetRequest.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisGetRequest.scala new file mode 100644 index 00000000..08ca3aea --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisGetRequest.scala @@ -0,0 +1,71 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.s2graph.core.GraphUtil + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ + +trait RedisRPC { + val key: Array[Byte] +} + +case class RedisGetRequest(k: Array[Byte]) extends RedisRPC { + /** + * For degree edge key(not sorted set key/value case) + */ + override val key = k + + assert(key.length > 0) + + var isIncludeDegree: Boolean = true + + var timeout: Long = _ + var count: Int = _ + var offset: Int = _ + + var min: Array[Byte] = _ + var minInclusive: Boolean = _ + var max: Array[Byte] = _ + var maxInclusive: Boolean = _ + + var minTime: Long = _ + var maxTime: Long = _ + + def setTimeout(time: Long): RedisGetRequest = { + this.timeout = time + this + } + def setCount(count: Int): RedisGetRequest = { + this.count = count + this + } + def setOffset(offset: Int): RedisGetRequest = { + this.offset = offset + this + } + + // for `interval` + def setFilter(min: Array[Byte], + minInclusive: Boolean, + max: Array[Byte], + maxInclusive: Boolean, + minTime: Long = -1, + maxTime: Long = -1): RedisGetRequest = { + + this.min = min; this.minInclusive = minInclusive; this.max = max; this.maxInclusive = maxInclusive; this.minTime = minTime; this.maxTime = maxTime; + this + } + + override def toString() = { + s"[RedisGetRequest] Key : ${GraphUtil.bytesToHexString(key)}" + } + +} + + +case class RedisSnapshotGetRequest(k: Array[Byte]) extends RedisRPC { + override val key = k + assert(key.length > 0) +} + diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeDeserializable.scala new file mode 100644 index 00000000..6224b637 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeDeserializable.scala @@ -0,0 +1,152 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.storage.StorageDeserializable._ +import org.apache.s2graph.core.mysqls.LabelMeta +import org.apache.s2graph.core.{Vertex, QueryParam, GraphUtil, IndexEdge} +import org.apache.s2graph.core.storage.{StorageDeserializable, Deserializable, CanSKeyValue, SKeyValue} +import org.apache.s2graph.core.types._ + +/** + * @author Junki Kim (wishoping@gmail.com) and Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +class RedisIndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] { + import StorageDeserializable._ + + type QualifierRaw = (Array[(Byte, InnerValLike)], // Index property key/value map + VertexId, // target vertex id + Byte, // Operation code + Boolean, // Whether or not target vertex id exists in Qualifier + Long, // timestamp + Int) // length of bytes read + type ValueRaw = (Array[(Byte, InnerValLike)], Int) + + /** + * Parse qualifier + * + * byte map + * [ qualifier length byte | indexed property count byte | indexed property values bytes | timestamp bytes | target id bytes | operation code byte ] + * + * - Please refer to https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/util/OrderedBytes.html regarding actual byte size of InnerVals. + * + * @param kv + * @param totalQualifierLen + * @param version + * @return + */ + private def parseQualifier(kv: SKeyValue, totalQualifierLen: Int, version: String): QualifierRaw = { + kv.operation match { + case SKeyValue.Increment => + (Array.empty[(Byte, InnerValLike)], VertexId(GraphType.DEFAULT_COL_ID, InnerVal.withStr("0", version)), GraphUtil.toOp("increment").get, true, 0, 0) + case _ => + var qualifierLen = 0 + var pos = 0 + val (idxPropsRaw, tgtVertexIdRaw, tgtVertexIdLen, timestamp) = { + val (props, endAt) = bytesToProps(kv.value, pos, version) + pos = endAt + qualifierLen += endAt + + // get timestamp value + val (tsInnerVal, numOfBytesUsed) = InnerVal.fromBytes(kv.value, pos, 0, version, false) + val ts = tsInnerVal.value match { + case n: BigDecimal => n.bigDecimal.longValue() + case _ => tsInnerVal.toString().toLong + } + + pos += numOfBytesUsed + qualifierLen += numOfBytesUsed + + val (tgtVertexId, tgtVertexIdLen) = + if (pos == totalQualifierLen) { // target id is included in qualifier + (VertexId(GraphType.DEFAULT_COL_ID, InnerVal.withStr("0", version)), 0) + } else { + TargetVertexId.fromBytes(kv.value, pos, kv.value.length, version) + } + pos += tgtVertexIdLen + qualifierLen += tgtVertexIdLen + (props, tgtVertexId, tgtVertexIdLen, ts) + } + val op = kv.value(totalQualifierLen) + pos += 1 + + (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, timestamp, pos) + } + } + + private def parseValue(kv: SKeyValue, offset: Int, version: String): ValueRaw = { + val value = + if ( kv.value.length > 0 ) kv.value.dropRight(1) + else kv.value + val (props, endAt) = bytesToKeyValues(value, offset, 0, version) + (props, endAt) + } + + private def parseDegreeValue(kv: SKeyValue, offset: Int, version: String): ValueRaw = { + (Array.fill[(Byte, InnerValLike)](1)(LabelMeta.degreeSeq -> InnerVal.withLong(Bytes.toLong(kv.value), version)), 0) + } + + /** + * Read first byte as qualifier length and check if length equals 0 + * + * @param kv + * @return true if first byte is zero value + */ + private def isDegree(kv: SKeyValue) = kv.operation == SKeyValue.Increment + + override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[IndexEdge]): IndexEdge = { + assert(_kvs.size == 1) + + val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) } + val kv = kvs.head + + val (srcVertexId, labelWithDir, labelIdxSeq, _, _) = cacheElementOpt.map { e => + (e.srcVertex.id, e.labelWithDir, e.labelIndexSeq, false, 0) + }.getOrElse(parseRow(kv, version)) + + val totalQualifierLen = + if (kv.value.length > 0 ) kv.value(kv.value.length-1).toInt + else 0 + + val (idxPropsRaw, tgtVertexIdRaw, op, isTgtVertexIdInQualifier, timestamp, numBytesRead) = + parseQualifier(kv, totalQualifierLen, version) + + // get non-indexed property key/ value + val (props, _) = if (op == GraphUtil.operations("incrementCount")) { + val (amount, _) = InnerVal.fromBytes(kv.value, numBytesRead, 0, version) + val countVal = amount.value.toString.toLong + val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version)) + (dummyProps, 8) + } else if ( isDegree(kv) ) { + parseDegreeValue(kv, numBytesRead, version) + } else { + // non-indexed property key/ value retrieval + + parseValue(kv, numBytesRead, version) + } + + val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException("invalid index seq")) + + val idxProps = for { + (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw) + } yield { + if (k == LabelMeta.degreeSeq) k -> v + else seq -> v + } + + val idxPropsMap = idxProps.toMap + val tgtVertexId = if (isTgtVertexIdInQualifier) { + idxPropsMap.get(LabelMeta.toSeq) match { + case None => tgtVertexIdRaw + case Some(vId) => TargetVertexId(GraphType.DEFAULT_COL_ID, vId) + } + } else tgtVertexIdRaw + + val _mergedProps = (idxProps ++ props).toMap + val mergedProps = + if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps + else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(timestamp, version)) + + IndexEdge(Vertex(srcVertexId, timestamp), Vertex(tgtVertexId, timestamp), labelWithDir, op, timestamp, labelIdxSeq, mergedProps) + + } +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeSerializable.scala new file mode 100644 index 00000000..353a36ea --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisIndexEdgeSerializable.scala @@ -0,0 +1,57 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.mysqls.LabelMeta +import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable} +import org.apache.s2graph.core.types.VertexId +import org.apache.s2graph.core.types.v2.InnerVal +import org.apache.s2graph.core.{GraphUtil, IndexEdge} + +/** + * @author Junki Kim (wishoping@gmail.com) and Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +class RedisIndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] { + import StorageSerializable._ + + val label = indexEdge.label + + val idxPropsMap = indexEdge.orders.toMap + val idxPropsBytes = propsToBytes(indexEdge.orders) + + override def toKeyValues: Seq[SKeyValue] = { + val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes.drop(GraphUtil.bytesForMurMurHash) + val labelWithDirBytes = indexEdge.labelWithDir.bytes + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(indexEdge.labelIndexSeq, isSnapshot = false) + val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) + val tgtIdBytes = VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes + + /** + * Qualifier and value byte array map + * + * * byte field design + * [{ # of index property - 1 byte } | - + * { series of index property values - sum of length with each property values bytes } | - + * { timestamp - 8 bytes } | { target id inner value - length of target id inner value bytes } | - + * { operation code byte - 1 byte } - + * { series of non-index property values - sum of length with each property values bytes } | - + * { qualifier total length - 1 byte }] + * + * ** !Serialize operation code byte after target id or series of index props bytes + */ + val timestamp = InnerVal(BigDecimal(indexEdge.ts)).bytes + val qualifier = + (idxPropsMap.get(LabelMeta.toSeq) match { + case None => Bytes.add(idxPropsBytes, timestamp, tgtIdBytes) + case Some(vId) => Bytes.add(idxPropsBytes, timestamp) + }) ++ Array.fill(1)(indexEdge.op) + + val qualifierLen = Array.fill[Byte](1)(qualifier.length.toByte) + val propsKv = propsToKeyValues(indexEdge.metas.toSeq) + + val value = qualifier ++ propsKv ++ qualifierLen + val emptyArray = Array.empty[Byte] + val kv = SKeyValue(emptyArray, row, emptyArray, emptyArray, value, indexEdge.version) + + Seq(kv) + } +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeDeserializable.scala new file mode 100644 index 00000000..18cddb40 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeDeserializable.scala @@ -0,0 +1,99 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.mysqls.{LabelMeta, LabelIndex} +import org.apache.s2graph.core.types._ +import org.apache.s2graph.core._ +import org.apache.s2graph.core.storage.{StorageDeserializable, SKeyValue, CanSKeyValue, Deserializable} + +/** + * @author Junki Kim (wishoping@gmail.com) and Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +class RedisSnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] { + import StorageDeserializable._ + + override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge ={ + version match { + case GraphType.VERSION4 => fromKeyValuesInnerV4(queryParam, kvs, version, cacheElementOpt) + case _ => throw new GraphExceptions.UnsupportedVersionException(">> Redis storage can only support v4 - current schema version ${label.schemaVersion}") + } + + } + + def statusCodeWithOp(byte: Byte): (Byte, Byte) = { + val statusCode = byte >> 4 + val op = byte & ((1 << 4) - 1) + (statusCode.toByte, op.toByte) + } + + def fromKeyValuesInnerV4[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = { + val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) } + assert(kvs.size == 1) + + val kv = kvs.head.copy(operation = SKeyValue.SnapshotPut) // Snapshot puts have to be identifiable in Redis. + val schemaVer = queryParam.label.schemaVersion + val cellVersion = kv.timestamp + + /** rowKey */ + def parseRowV4(kv: SKeyValue, version: String) = { + var pos = -GraphUtil.bytesForMurMurHash + val (srcIdAndTgtId, srcIdAndTgtIdBytesLen) = SourceAndTargetVertexIdPair.fromBytes(kv.row, pos, kv.row.length, version) + pos += srcIdAndTgtIdBytesLen + val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) + pos += 4 + val (labelIdxSeq, isSnapshot) = bytesToLabelIndexSeqWithIsSnapshot(kv.row, pos) + + val rowLen = srcIdAndTgtIdBytesLen + 4 + 1 + (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isSnapshot, rowLen) + + } + val (srcInnerId, tgtInnerId, labelWithDir, _, _, _) = cacheElementOpt.map { e => + (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0) + }.getOrElse(parseRowV4(kv, schemaVer)) + + val srcVertexId = SourceVertexId(GraphType.DEFAULT_COL_ID, srcInnerId) + val tgtVertexId = SourceVertexId(GraphType.DEFAULT_COL_ID, tgtInnerId) + + val (props, op, _cellVersion, ts, statusCode, _pendingEdgeOpt) = { + var pos = 0 + val (tsInnerVal, numOfBytesUsed) = InnerVal.fromBytes(kv.value, pos, 0, schemaVer, false) + val version = tsInnerVal.value match { + case n: BigDecimal => n.bigDecimal.longValue() + case _ => tsInnerVal.toString().toLong + } + + pos += numOfBytesUsed + val (statusCode, op) = statusCodeWithOp(kv.value(pos)) + pos += 1 + val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer) + val kvsMap = props.toMap + val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong + + pos = endAt + val _pendingEdgeOpt = + if (pos == kv.value.length) None + else { + val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos)) + pos += 1 + val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer) + pos = endAt + val lockTs = Option(Bytes.toLong(kv.value, pos, 8)) + + val pendingEdge = + Edge(Vertex(srcVertexId, ts), + Vertex(tgtVertexId, ts), + labelWithDir, pendingEdgeOp, + ts, pendingEdgeProps.toMap, + statusCode = pendingEdgeStatusCode, lockTs = lockTs) + Option(pendingEdge) + } + + (kvsMap, op, version, ts, statusCode, _pendingEdgeOpt) + } + + SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), + labelWithDir, op, cellVersion, props, statusCode = statusCode, + pendingEdgeOpt = _pendingEdgeOpt, lockTs = None) + } + +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeSerializable.scala new file mode 100644 index 00000000..0bf12162 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisSnapshotEdgeSerializable.scala @@ -0,0 +1,58 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.mysqls.LabelIndex +import org.apache.s2graph.core.types.v2.InnerVal +import org.apache.s2graph.core.{GraphUtil, GraphExceptions, SnapshotEdge} +import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable} +import org.apache.s2graph.core.types.{SourceAndTargetVertexIdPair, GraphType} + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +class RedisSnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] { + import StorageSerializable._ + + val label = snapshotEdge.label + + override def toKeyValues: Seq[SKeyValue] = { + label.schemaVersion match { + case GraphType.VERSION4 => toKeyValuesInnerV4 + case _ => throw new GraphExceptions.UnsupportedVersionException(s">> Redis storage can only support v4: current schema version ${label.schemaVersion}") + } + } + + def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = { + val byte = (((statusCode << 4) | op).toByte) + Array.fill(1)(byte.toByte) + } + + def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op), + propsToKeyValuesWithTs(snapshotEdge.props.toList)) + + private def toKeyValuesInnerV4: Seq[SKeyValue] = { + val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes + val labelWithDirBytes = snapshotEdge.labelWithDir.bytes + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(LabelIndex.DefaultSeq, isSnapshot = true) + + val row = Bytes.add( + srcIdAndTgtIdBytes.drop(GraphUtil.bytesForMurMurHash), + labelWithDirBytes, + labelIndexSeqWithIsInvertedBytes + ) + + val timestamp = InnerVal(BigDecimal(snapshotEdge.version)).bytes + + val value = snapshotEdge.pendingEdgeOpt match { + case None => Bytes.add(timestamp, valueBytes()) + case Some(pendingEdge) => + val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op) + val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq) + val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get) + timestamp ++ valueBytes() ++ opBytes ++ propsBytes ++ lockBytes + } + + val kv = SKeyValue(Array.empty[Byte], row, Array.empty[Byte], Array.empty[Byte], value, snapshotEdge.version, operation = SKeyValue.SnapshotPut) + Seq(kv) + } +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisStorage.scala new file mode 100644 index 00000000..a69a5247 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisStorage.scala @@ -0,0 +1,654 @@ +package com.kakao.s2graph.core.storage.redis + +import java.util.concurrent.TimeUnit + +import com.google.common.cache.CacheBuilder +import com.typesafe.config.Config +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.GraphExceptions.UnsupportedVersionException +import org.apache.s2graph.core._ +import org.apache.s2graph.core.mysqls.LabelMeta +import org.apache.s2graph.core.storage.redis._ +import org.apache.s2graph.core.storage.redis.jedis.JedisClient +import org.apache.s2graph.core.storage.{CanSKeyValue, SKeyValue, Storage} +import org.apache.s2graph.core.types._ +import org.apache.s2graph.core.utils.logger +import redis.clients.jedis.Jedis + +import scala.collection.JavaConversions._ +import scala.collection.Seq +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.util.hashing.MurmurHash3 +import scala.util.{Failure, Success} + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +class RedisStorage(override val config: Config)(implicit ec: ExecutionContext) + extends Storage[Future[QueryRequestWithResult]](config) { + + import GraphType._ + + val futureCache = CacheBuilder.newBuilder() + .initialCapacity(maxSize) + .concurrencyLevel(Runtime.getRuntime.availableProcessors()) + .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS) + .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS) + .maximumSize(maxSize).build[java.lang.Long, (Long, Future[QueryRequestWithResult])]() + + /** Simple Vertex Cache */ + private val vertexCache = CacheBuilder.newBuilder() + .initialCapacity(maxSize) + .concurrencyLevel(Runtime.getRuntime.availableProcessors()) + .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS) + .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS) + .maximumSize(maxSize).build[java.lang.Integer, Option[Vertex]]() + + override def indexEdgeDeserializer(schemaVer: String) = + schemaVer match { + case VERSION4 => new RedisIndexEdgeDeserializable + case _ => throw new UnsupportedVersionException(s"Redis storage does not support version ${schemaVer}") + } + + override def snapshotEdgeDeserializer(schemaVer: String) = + schemaVer match { + case VERSION4 => new RedisSnapshotEdgeDeserializable + case _ => throw new UnsupportedOperationException + } + + override val vertexDeserializer = new RedisVertexDeserializable + + override def indexEdgeSerializer(indexedEdge: IndexEdge) = new RedisIndexEdgeSerializable(indexedEdge) + + override def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge) = new RedisSnapshotEdgeSerializable(snapshotEdge) + + override def vertexSerializer(vertex: Vertex) = new RedisVertexSerializable(vertex) + + private val RedisZsetScore = 1 + + private val client = new JedisClient(config) + + /** + * decide how to store given SKeyValue into storage using storage's client. + * we assumes that each storage implementation has client as member variable. + * + * ex) Asynchbase client provide PutRequest/DeleteRequest/AtomicIncrement/CompareAndSet operations + * to actually apply byte array into storage. in this case, AsynchbaseStorage use HBaseClient + * and build + fire rpc and return future that will return if this rpc has been succeed. + * + // * @param kv : SKeyValue that need to be stored in storage. +// * @param withWait : flag to control wait ack from storage. + * note that in AsynchbaseStorage(which support asynchronous operations), even with true, + * it never block thread, but rather submit work and notified by event loop when storage send ack back. + * @return ack message from storage. + */ + override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = { + if (kvs.isEmpty) { + Future.successful(true) + } else { + val futures = kvs.map { kv => writeToRedis(kv, withWait) } + Future.sequence(futures).map(_.forall(identity)) + } + } + + def removeWithQualifier(jedis: Jedis, kv: SKeyValue): Boolean = { + val qualifierPos = kv.value(kv.value.length - 1).toInt - 2 // qualifier length w/o op byte + val qualifierWithoutOp = kv.value.slice(0, qualifierPos) + val min = Bytes.toBytes("[") ++ qualifierWithoutOp ++ Array.fill[Byte](1)(0) + val max = Bytes.toBytes("[") ++ qualifierWithoutOp ++ Array.fill[Byte](1)(-1) + jedis.zremrangeByLex(kv.row, min, max) > 0 + } + + def writeToRedis(kv: SKeyValue, withWait: Boolean): Future[Boolean] = { + val future = Future.successful { + client.doBlockWithKey[Boolean](GraphUtil.bytesToHexString(kv.row)) { jedis => + kv.operation match { + case SKeyValue.SnapshotPut => + jedis.set(kv.row, kv.value) == 1 + + // vertex write + case SKeyValue.Put if kv.qualifier.length > 0 => + // disregard vertex properties at overwrite, but only once at the beginning of each insert + if (kv.timestamp == 0) jedis.del(kv.row) + jedis.zadd(kv.row, RedisZsetScore, kv.qualifier ++ kv.value) == 1 + + // edge write + case SKeyValue.Put if kv.qualifier.length == 0 => + jedis.zadd(kv.row, RedisZsetScore, kv.value) == 1 + + // vertex delete + case SKeyValue.Delete if kv.qualifier.length > 0 => + jedis.zrem(kv.row, kv.qualifier ++ kv.value) == 1 + + // edge delete + case SKeyValue.Delete if kv.qualifier.length == 0 => + removeWithQualifier(jedis, kv) + + case SKeyValue.Increment => true // no need for degree increment since Redis storage uses ZCARD for degree + } + } match { + case Success(b) => b + case Failure(e) => + logger.error(s"mutation failed. $kv", e) + false + } + } + + if (withWait) future else Future.successful(true) + } + + /** Delete All */ + override protected def deleteAllFetchedEdgesAsyncOld(queryRequest: QueryRequest, + queryResult: QueryResult, + requestTs: Long, + retryNum: Int): Future[Boolean] = { + val queryParam = queryRequest.queryParam + val zkQuorum = queryParam.label.hbaseZkAddr + val futures = for { + edgeWithScore <- queryResult.edgeWithScoreLs + (edge, score) = EdgeWithScore.unapply(edgeWithScore).get + } yield { + /** reverted direction */ + val reversedIndexedEdgesMutations = edge.duplicateEdge.edgesWithIndex.flatMap { indexEdge => + indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++ + buildIncrementsAsync(indexEdge, -1L) + } + val reversedSnapshotEdgeMutations = snapshotEdgeSerializer(edge.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.SnapshotPut)) + val forwardIndexedEdgeMutations = edge.edgesWithIndex.flatMap { indexEdge => + indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++ + buildIncrementsAsync(indexEdge, -1L) + } + val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations + writeToStorage(zkQuorum, mutations, withWait = true) + } + + Future.sequence(futures).map { rets => rets.forall(identity) } + } + + /** + * create table on storage. + * if storage implementation does not support namespace or table, then there is nothing to be done + * + * @param zkAddr + * @param tableName + * @param cfs + * @param regionMultiplier + * @param ttl + * @param compressionAlgorithm + */ + override def createTable(zkAddr: String, tableName: String, cfs: List[String], regionMultiplier: Int, ttl: Option[Int], compressionAlgorithm: String): Unit = { + logger.info(s"create table is not supported") + } + + /** + * build proper request which is specific into storage to call fetchIndexEdgeKeyValues or fetchSnapshotEdgeKeyValues. + * for example, Asynchbase use GetRequest, Scanner so this method is responsible to build + * client request(GetRequest, Scanner) based on user provided query. + * + * @param queryRequest + * @return + */ + + override def buildRequest(queryRequest: QueryRequest): RedisRPC = { + val srcVertex = queryRequest.vertex + + val queryParam = queryRequest.queryParam + val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt + val label = queryParam.label + val labelWithDir = queryParam.labelWithDir + val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir) + val (srcInnerId, tgtInnerId) = tgtVertexIdOpt match { + case Some(tgtVertexId) => // _to is given. + /** we use toSnapshotEdge so dont need to swap src, tgt */ + val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion) + val tgt = InnerVal.convertVersion(tgtVertexId, tgtColumn.columnType, label.schemaVersion) + (src, tgt) + case None => + val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion) + (src, src) + } + + val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId)) + val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId)) + val currentTs = System.currentTimeMillis() + val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs)) + val edge = Edge(srcV, tgtV, labelWithDir, propsWithTs = propsWithTs) + + val (kv, isSnapshot) = if (tgtVertexIdOpt.isDefined) { + val snapshotEdge = edge.toSnapshotEdge + (snapshotEdgeSerializer(snapshotEdge).toKeyValues.head, true) + } else { + val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == queryParam.labelOrderSeq) + assert(indexedEdgeOpt.isDefined) + + val indexedEdge = indexedEdgeOpt.get + (indexEdgeSerializer(indexedEdge).toKeyValues.head, false) + } + + // Redis supports client-side sharding and does not require hash key so remove heading hash key(2 bytes) + val rowkey = kv.row + + // 1. RedisGet instance initialize + if (isSnapshot) new RedisSnapshotGetRequest(rowkey) + else { + val _get = new RedisGetRequest(rowkey) + _get.isIncludeDegree = !tgtVertexIdOpt.isDefined + + // 2. set filter and min/max value's key build + val (minTs, maxTs) = queryParam.duration.getOrElse(-1L -> -1L) + val (min, max) = + if (queryParam.columnRangeFilterMinBytes.length != 0 && queryParam.columnRangeFilterMaxBytes.length != 0) + (Bytes.add(Bytes.toBytes("["), queryParam.columnRangeFilterMinBytes), + Bytes.add(Bytes.toBytes("["), queryParam.columnRangeFilterMaxBytes)) + else + ("-".getBytes, "+".getBytes) + + val (newLimit, newOffset) = revertLimit(queryParam.offset, queryParam.limit) + + _get.setCount(newLimit) + .setOffset(newOffset) + .setTimeout(queryParam.rpcTimeoutInMillis) + .setFilter(min, true, max, true, minTs, maxTs) + } + } + + // Degrees are handled with separate keys in Redis. + // No need for offset/ limit adjustments in QueryParam limit(). + // Hence, the function name revertLimit. + private def revertLimit(offset: Int, limit: Int): (Int, Int) = { + /** since degree info is located on first always */ + val (newLimit, newOffset) = if (offset == 0) { + (limit - 1, offset) + } else { + (limit, offset - 1) + } + (newLimit, newOffset) + } + + private def fetchKeyValuesInner(request: RedisRPC) = { + Future[Seq[SKeyValue]] { + // send rpc call to Redis instance + client.doBlockWithKey[Seq[SKeyValue]](GraphUtil.bytesToHexString(request.key)) { jedis => + val paddedBytes = Array.fill[Byte](2)(0) + request match { + case req@RedisGetRequest(_) => + val result = jedis.zrangeByLex(req.key, req.min, req.max, req.offset, req.count).toSeq.map(v => + SKeyValue(Array.empty[Byte], paddedBytes ++ req.key, Array.empty[Byte], Array.empty[Byte], v, 0L) + ) + if (req.isIncludeDegree) { + val degree = jedis.zcard(req.key) + val degreeBytes = Bytes.toBytes(degree) + SKeyValue(Array.empty[Byte], paddedBytes ++ req.key, Array.empty[Byte], Array.empty[Byte], degreeBytes, 0L, operation = SKeyValue.Increment) +: result + } else result + case req@RedisSnapshotGetRequest(_) => + val _result = jedis.get(req.key) + if (_result == null) { + Seq.empty[SKeyValue] + } + else { + val (tsInnerVal, numOfBytesUsed) = InnerVal.fromBytes(_result, 0, 0, GraphType.VERSION4, false) + + val ts = tsInnerVal.value match { + case n: BigDecimal => n.bigDecimal.longValue() + case _ => tsInnerVal.toString().toLong + } + + val snapshot = SKeyValue(Array.empty[Byte], req.key, Array.empty[Byte], Array.empty[Byte], _result, ts, operation = SKeyValue.SnapshotPut) + Seq[SKeyValue](snapshot) + } + } + } match { + case Success(v) => v + case Failure(e) => + // logger.error(s">> get fail!! $e") + e.printStackTrace() + Seq[SKeyValue]() + } + } + } + + /** + * fetch IndexEdges for given queryParam in queryRequest. + * this expect previous step starting score to propagate score into next step. + * also parentEdges is necessary to return full bfs tree when query require it. + * + * note that return type is general type. + * for example, currently we wanted to use Asynchbase + * so single I/O return type should be Deferred[T]. + * + * if we use native hbase client, then this return type can be Future[T] or just T. + * + * @param queryRequest + * @param prevStepScore + * @param isInnerCall + * @param parentEdges + * @return + */ + override def fetch(queryRequest: QueryRequest, + prevStepScore: Double, + isInnerCall: Boolean, + parentEdges: Seq[EdgeWithScore]): Future[QueryRequestWithResult] = { + def fetchInner(request: RedisRPC) = { + fetchKeyValuesInner(request).map { values => + val edgeWithScores = toEdges(values, queryRequest.queryParam, prevStepScore, isInnerCall, parentEdges) + val resultEdgesWithScores = if (queryRequest.queryParam.sample >= 0) { + sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample) + } else edgeWithScores + QueryRequestWithResult(queryRequest, QueryResult(resultEdgesWithScores)) + }.recover { case ex: Exception => + logger.error(s"fetchInner failed. fallback return. $request}", ex) + QueryRequestWithResult(queryRequest, QueryResult(isFailure = true)) + } + } + + def checkAndExpire(request: RedisRPC, + cacheKey: Long, + cacheTTL: Long, + cachedAt: Long, + defer: Future[QueryRequestWithResult]): Future[QueryRequestWithResult] = { + + if (System.currentTimeMillis() >= cachedAt + cacheTTL) { + // future is too old. so need to expire and fetch new data from storage. + futureCache.asMap().remove(cacheKey) + val newPromise = Promise[QueryRequestWithResult]() + val newFuture = newPromise.future + futureCache.asMap().putIfAbsent(cacheKey, (System.currentTimeMillis(), newFuture)) match { + case null => + // only one thread succeed to come here concurrently + // initiate fetch to storage then add callback on complete to finish promise. + fetchInner(request) map { queryRequestWithResult => + newPromise.trySuccess(queryRequestWithResult) + queryRequestWithResult + } + newFuture + case (cachedAt, oldDefer) => oldDefer + } + } else { + // future is not too old so reuse it. + defer + } + } + + val queryParam = queryRequest.queryParam + val cacheTTL = queryParam.cacheTTLInMillis + val request = buildRequest(queryRequest) + if (cacheTTL <= 0) fetchInner(request) + else { + val cacheKeyBytes = Bytes.add(queryRequest.query.cacheKeyBytes, toCacheKeyBytes(request)) + val cacheKey = queryParam.toCacheKey(cacheKeyBytes) + + val cacheVal = futureCache.getIfPresent(cacheKey) + cacheVal match { + case null => + // here there is no promise set up for this cacheKey so we need to set promise on future cache. + val promise = Promise[QueryRequestWithResult]() + val future = promise.future + val now = System.currentTimeMillis() + val (cachedAt, defer) = futureCache.asMap().putIfAbsent(cacheKey, (now, future)) match { + case null => + fetchInner(request) map { queryRequestWithResult => + promise.trySuccess(queryRequestWithResult) + queryRequestWithResult + } + (now, future) + case oldVal => oldVal + } + checkAndExpire(request, cacheKey, cacheTTL, cachedAt, defer) + case (cachedAt, defer) => + checkAndExpire(request, cacheKey, cacheTTL, cachedAt, defer) + } + } + + } + + override def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = { + def fromResult(queryParam: QueryParam, + kvs: Seq[SKeyValue], + version: String): Option[Vertex] = { + + if (kvs.isEmpty) None + else { + vertexDeserializer.fromKeyValues(queryParam, kvs, version, None) + } + } + + val futures = vertices.map { vertex => + val kvs = vertexSerializer(vertex).toKeyValues + val get = new RedisGetRequest(kvs.head.row) + get.isIncludeDegree = false + + val cacheKey = MurmurHash3.stringHash(get.toString) + val cacheVal = vertexCache.getIfPresent(cacheKey) + if (cacheVal == null) { + val result = client.doBlockWithKey[Set[SKeyValue]](GraphUtil.bytesToHexString(get.key)) { jedis => + get.setFilter("-".getBytes, true, "+".getBytes, true) + jedis.zrangeByLex(get.key, get.min, get.max).toSet[Array[Byte]].map(v => + SKeyValue(Array.empty[Byte], get.key, Array.empty[Byte], Array.empty[Byte], v, 0L) + ) + } match { + case Success(v) => + v + case Failure(e) => + logger.error(s"Redis vertex get fail: ", e) + Set[SKeyValue]() + } + val fetchVal = fromResult(QueryParam.Empty, result.toSeq, vertex.serviceColumn.schemaVersion) + Future.successful(fetchVal) + } + + else Future.successful(cacheVal) + } + + Future.sequence(futures).map { result => result.toList.flatten } + } + + /** + * write requestKeyValue into storage if the current value in storage that is stored matches. + * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge. + * + * Most important thing is this have to be 'atomic' operation. + * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be + * either blocked or failed on write-write conflict case. + * + * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to + * prevent wrong data for read. + * + * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction, + * compareAndSet to synchronize. + * + * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'. + * for storage that does not support concurrency control, then storage implementation + * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues) + * and write(writeLock). + * + * @param requestKeyValue + * @param expectedOpt + * @return + */ + override def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean] = { + Future.successful { + client.doBlockWithKey[Boolean](GraphUtil.bytesToHexString(requestKeyValue.row)) { jedis => + try { + expectedOpt match { + case Some(expected) => + + jedis.watch(requestKeyValue.row) + val curVal = jedis.get(requestKeyValue.row) + val result = + if (Bytes.compareTo(expected.value, curVal) == 0) { + val transaction = jedis.multi() + try { + transaction.set(requestKeyValue.row, requestKeyValue.value) + transaction.exec() + } catch { + case e: Throwable => + logger.error(s">> error thrown", e) + transaction.discard() + false + } + } else "[FAIL]" + + result != null && result.toString.equals("[OK]") + + case None => + + jedis.watch(requestKeyValue.row) + val curVal = jedis.get(requestKeyValue.row) + + val result = + if (curVal == null) { + val transaction = jedis.multi() + try { + transaction.set(requestKeyValue.row, requestKeyValue.value) + transaction.exec() + } catch { + case e: Throwable => + logger.error(s">> error thrown", e) + transaction.discard() + false + } + + } else { + logger.error(s"\n[[ writeLock failed") + "[FAIL]" + } + + result != null && result.toString.equals("[OK]") + } + } catch { + case ex: Throwable => + logger.error(s"writeLock transaction failed old : $requestKeyValue, expected : $expectedOpt", ex) + throw ex + } + } match { + case Success(b) => b + case Failure(e) => + logger.error(s"writeLock failed old : $requestKeyValue, expected : $expectedOpt", e) + false + } + } + } + + def simpleWrite(k: String, v: String): Boolean = { + client.doBlockWithKey(k) { jedis => + val result = jedis.set(k, v) + result != null && result.toString.equals("[OK]") + } match { + case Success(b) => b + case Failure(e) => + logger.error("write failed") + false + } + } + + def writeWithTx(k: String, v: String, exp: String): Future[Boolean] = { + Future.successful { + client.doBlockWithKey(k) { jedis => + jedis.watch(k) + val fetched = jedis.get(k) + // logger.info(s"fetched: $fetched") + val result = if (fetched.contentEquals(exp)) { + val tx = jedis.multi() + try { + tx.set(k, v) + val r = tx.exec() + r + } catch { + case e: Throwable => + tx.discard() + false + } + } else "[FAIL]" + result != null && result.toString.equals("[OK]") + + } match { + case Success(b) => b + case Failure(e) => + logger.error(s"write failed: key - $k, val - $v, exp - $exp") + false + } + } + } + + + /** + * this method need to be called when client shutdown. this is responsible to cleanUp the resources + * such as client into storage. + */ + override def flush(): Unit = {} + + /** + * fetch SnapshotEdge for given request from storage. + * also storage datatype should be converted into SKeyValue. + * note that return type is Sequence rather than single SKeyValue for simplicity, + * even though there is assertions sequence.length == 1. + * + * @param request + * @return + */ + override def fetchSnapshotEdgeKeyValues(request: AnyRef): Future[Seq[SKeyValue]] = { + val defer = fetchKeyValuesInner(request.asInstanceOf[RedisRPC]) + defer.map { kvsArr => + kvsArr.map { kv => + implicitly[CanSKeyValue[SKeyValue]].toSKeyValue(kv) + } + } + } + + /** + * decide how to apply given edges(indexProps values + Map(_count -> countVal)) into storage. + * + * @param edges + * @param withWait + * @return + */ + override def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = { + logger.error(s"'incrementCount' operation is not yet supported") + Future[Seq[(Boolean, Long)]] { + Seq[(Boolean, Long)]() + } + } + + /** + * responsible to fire parallel fetch call into storage and create future that will return merged result. + * + * @param queryRequestWithScoreLs + * @param prevStepEdges + * @return + */ + override def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)], + prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]] = { + + val reads: Seq[Future[QueryRequestWithResult]] = for { + (queryRequest, prevStepScore) <- queryRequestWithScoreLs + } yield { + val prevStepEdgesOpt = prevStepEdges.get(queryRequest.vertex.id) + if (prevStepEdgesOpt.isEmpty) throw new RuntimeException("miss match on prevStepEdge and current GetRequest") + + val parentEdges = for { + parentEdge <- prevStepEdgesOpt.get + } yield parentEdge + + fetch(queryRequest, prevStepScore, isInnerCall = true, parentEdges) + } + + Future.sequence(reads) + } + + private def toCacheKeyBytes(redisRpc: RedisRPC): Array[Byte] = { + redisRpc match { + case getRequest: RedisGetRequest => getRequest.key + case snapshotRequest: RedisSnapshotGetRequest => snapshotRequest.key + case _ => + logger.error(s"toCacheKeyBytes failed. not supported class type. $redisRpc") + Array.empty[Byte] + } + } + + /** + * fetch Vertex for given request from storage. + * @param request + * @return + */ + override def fetchVertexKeyValues(request: AnyRef): Future[scala.Seq[SKeyValue]] = ??? +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexDeserializable.scala new file mode 100644 index 00000000..a0137ae0 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexDeserializable.scala @@ -0,0 +1,49 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.storage.CanSKeyValue +import org.apache.s2graph.core.storage.serde.vertex.VertexDeserializable +import org.apache.s2graph.core.types.{InnerVal, InnerValLike, VertexId} +import org.apache.s2graph.core.{GraphUtil, QueryParam, Vertex} + +import scala.collection.mutable.ListBuffer + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ + +class RedisVertexDeserializable extends VertexDeserializable { + + override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[Vertex]): Vertex = { + val _kvs = kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) } + + val kv = _kvs.head + val (vertexId, _) = VertexId.fromBytes(kv.row, -GraphUtil.bytesForMurMurHash, kv.row.length, version) // no hash bytes => offset: -2 + + var maxTs = Long.MinValue + val propsMap = new collection.mutable.HashMap[Int, InnerValLike] + val belongLabelIds = new ListBuffer[Int] + + for { + kv <- _kvs + } { + var offset = 0 + val propKey = Bytes.toInt(kv.value) + offset += 4 + val ts = Bytes.toLong(kv.value, offset, 8) + offset += 8 + if (ts > maxTs) maxTs = ts + + if (Vertex.isLabelId(propKey)) { + belongLabelIds += Vertex.toLabelId(propKey) + } else { + val (value, _) = InnerVal.fromBytes(kv.value, offset, kv.value.length, version) + propsMap += (propKey -> value) + } + } + assert(maxTs != Long.MinValue) + Vertex(vertexId, maxTs, propsMap.toMap, belongLabelIds = belongLabelIds) + + } +} + diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexSerializable.scala new file mode 100644 index 00000000..2009af26 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/RedisVertexSerializable.scala @@ -0,0 +1,21 @@ +package org.apache.s2graph.core.storage.redis + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.storage.{SKeyValue, Serializable} +import org.apache.s2graph.core.{GraphUtil, Vertex} + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/19. + */ +case class RedisVertexSerializable(vertex: Vertex) extends Serializable[Vertex] { + override def toKeyValues: Seq[SKeyValue] = { + val row = vertex.id.bytes.drop(GraphUtil.bytesForMurMurHash) + val base = for ((k, v) <- vertex.props ++ vertex.defaultProps) yield Bytes.toBytes(k) -> v.bytes + val belongsTo = vertex.belongLabelIds.map { labelId => Bytes.toBytes(Vertex.toPropKey(labelId)) -> Array.empty[Byte] } + val emptyArray = Array.empty[Byte] + (base ++ belongsTo).toIndexedSeq.zipWithIndex.map { case ((qualifier, value), index) => + val qualifierWithTs = qualifier ++ Bytes.toBytes(vertex.ts) + SKeyValue(emptyArray, row, emptyArray, qualifierWithTs, value, index) + } toSeq + } +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/jedis/JedisClient.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/jedis/JedisClient.scala new file mode 100644 index 00000000..d5fdf91d --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/redis/jedis/JedisClient.scala @@ -0,0 +1,93 @@ +package org.apache.s2graph.core.storage.redis.jedis + +import com.typesafe.config.Config +import org.apache.s2graph.core.GraphUtil +import org.slf4j.LoggerFactory +import redis.clients.jedis._ +import redis.clients.jedis.exceptions.JedisException + +import scala.collection.JavaConversions._ +import scala.util.Try + +/** + * @author Junki Kim (wishoping@gmail.com) and Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Jan/07. + */ +class JedisClient(config: Config) { + // TODO: refactor me + lazy val instances: List[(String, Int)] = if (config.hasPath("redis.storage")) { + (for { + s <- config.getStringList("redis.storage") + } yield { + val sp = s.split(':') + (sp(0), if (sp.length > 1) sp(1).toInt else 6379) + }).toList + } else List("localhost" -> 6379) + + + private val log = LoggerFactory.getLogger(getClass) + + val poolConfig = new JedisPoolConfig() + poolConfig.setMaxTotal(150) + poolConfig.setMaxIdle(100) + poolConfig.setMaxWaitMillis(200) + + val jedisPools = instances.map { case (host, port) => +// log.info(s">> jedisPool initialized : $host, $port") + new JedisPool(poolConfig, host, port) + } + + def getBucketIdx(key: String): Int = { + GraphUtil.murmur3(key) % jedisPools.size + } + + def doBlockWithIndex[T](idx: Int)(f: Jedis => T): Try[T] = { + Try { + val pool = jedisPools(idx) + + var jedis: Jedis = null + + try { + jedis = pool.getResource + + f(jedis) + } + catch { + case e: JedisException => + pool.returnBrokenResource(jedis) + + jedis = null + throw e + } + finally { + if (jedis != null) { + pool.returnResource(jedis) + } + } + } + } + + def doBlockWithKey[T](key: String)(f: Jedis => T): Try[T] = { + doBlockWithIndex(getBucketIdx(key))(f) + } + +} + +class JedisTransaction(passClient: Client) extends Transaction(passClient) { + + def evalWithLong(script: Array[Byte], keys: List[Array[Byte]], args: List[Array[Byte]]): Response[java.lang.Long] = { + val params: Array[Array[Byte]] = (keys ++ args).toArray + + getClient(script).eval(script, Protocol.toByteArray(keys.length), params) + + return getResponse(BuilderFactory.LONG) + } + + def evalWithString(script: Array[Byte], keys: List[Array[Byte]], args: List[Array[Byte]]): Response[java.lang.String] = { + val params: Array[Array[Byte]] = (keys ++ args).toArray + + getClient(script).eval(script, Protocol.toByteArray(keys.length), params) + + return getResponse(BuilderFactory.STRING) + } + +} diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala index f2339402..f665fd12 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala @@ -36,7 +36,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte // val degree = Bytes.toLong(kv.value) val degree = bytesToLongFunc(kv.value, 0) val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version)) - val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) + val tgtVertexIdRaw = VertexId(GraphType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0) } @@ -48,7 +48,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte pos = endAt qualifierLen += endAt val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) { - (HBaseType.defaultTgtVertexId, 0) + (GraphType.defaultTgtVertexId, 0) } else { TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version) } @@ -90,7 +90,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte pos += srcIdLen val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) pos += 4 - val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsSnapshot(kv.row, pos) pos += 1 val op = kv.row(pos) @@ -103,7 +103,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte val ts = kv.timestamp val props = Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, version), LabelMeta.degreeSeq -> InnerVal.withLong(degreeVal, version)) - val tgtVertexId = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) + val tgtVertexId = VertexId(GraphType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, props) } else { // not degree edge @@ -112,7 +112,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte val (idxPropsRaw, endAt) = bytesToProps(kv.row, pos, version) pos = endAt val (tgtVertexIdRaw, tgtVertexIdLen) = if (endAt == kv.row.length) { - (HBaseType.defaultTgtVertexId, 0) + (GraphType.defaultTgtVertexId, 0) } else { TargetVertexId.fromBytes(kv.row, endAt, kv.row.length, version) } @@ -126,7 +126,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte val tgtVertexId = idxPropsMap.get(LabelMeta.toSeq) match { case None => tgtVertexIdRaw - case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId) + case Some(vId) => TargetVertexId(GraphType.DEFAULT_COL_ID, vId) } val (props, _) = if (op == GraphUtil.operations("incrementCount")) { diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala index 116412ce..847e2a47 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala @@ -39,7 +39,7 @@ class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge override def toKeyValues: Seq[SKeyValue] = { val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes val labelWithDirBytes = indexEdge.labelWithDir.bytes - val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false) + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(indexEdge.labelIndexSeq, isSnapshot = false) val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}") diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala index d8bef976..618b0a06 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala @@ -35,7 +35,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte // val degree = Bytes.toLong(kv.value) val degree = bytesToLongFunc(kv.value, 0) val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version)) - val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) + val tgtVertexIdRaw = VertexId(GraphType.DEFAULT_COL_ID, InnerVal.withStr("0", version)) (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0) } @@ -47,7 +47,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte pos = endAt qualifierLen += endAt val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) { - (HBaseType.defaultTgtVertexId, 0) + (GraphType.defaultTgtVertexId, 0) } else { TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version) } @@ -116,7 +116,7 @@ class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = byte val tgtVertexId = if (tgtVertexIdInQualifier) { idxPropsMap.get(LabelMeta.toSeq) match { case None => tgtVertexIdRaw - case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId) + case Some(vId) => TargetVertexId(GraphType.DEFAULT_COL_ID, vId) } } else tgtVertexIdRaw diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala index 189de225..89d63c2a 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala @@ -38,7 +38,7 @@ class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge override def toKeyValues: Seq[SKeyValue] = { val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes val labelWithDirBytes = indexEdge.labelWithDir.bytes - val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false) + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(indexEdge.labelIndexSeq, isSnapshot = false) val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}") diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala index 0380ec93..206f57ef 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta} import org.apache.s2graph.core.storage.StorageDeserializable._ import org.apache.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue, StorageDeserializable} -import org.apache.s2graph.core.types.{HBaseType, LabelWithDirection, SourceAndTargetVertexIdPair, SourceVertexId} +import org.apache.s2graph.core.types.{GraphType, LabelWithDirection, SourceAndTargetVertexIdPair, SourceVertexId} import org.apache.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex} class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] { @@ -51,7 +51,7 @@ class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] { pos += srcIdAndTgtIdLen val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) pos += 4 - val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsSnapshot(kv.row, pos) val rowLen = srcIdAndTgtIdLen + 4 + 1 (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isInverted, rowLen) @@ -61,8 +61,8 @@ class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] { (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0) }.getOrElse(parseRowV3(kv, schemaVer)) - val srcVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, srcInnerId) - val tgtVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, tgtInnerId) + val srcVertexId = SourceVertexId(GraphType.DEFAULT_COL_ID, srcInnerId) + val tgtVertexId = SourceVertexId(GraphType.DEFAULT_COL_ID, tgtInnerId) val (props, op, ts, statusCode, _pendingEdgeOpt) = { var pos = 0 diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala index 716a6b9a..3047ac82 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala @@ -43,7 +43,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[ override def toKeyValues: Seq[SKeyValue] = { val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes val labelWithDirBytes = snapshotEdge.labelWithDir.bytes - val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true) + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(LabelIndex.DefaultSeq, isSnapshot = true) val row = Bytes.add(srcIdAndTgtIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala index 2eb2b1bd..c05e260d 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala @@ -48,7 +48,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[ override def toKeyValues: Seq[SKeyValue] = { val srcIdBytes = VertexId.toSourceVertexId(snapshotEdge.srcVertex.id).bytes val labelWithDirBytes = snapshotEdge.labelWithDir.bytes - val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true) + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsSnapshot(LabelIndex.DefaultSeq, isSnapshot = true) val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) val tgtIdBytes = VertexId.toTargetVertexId(snapshotEdge.tgtVertex.id).bytes diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/GraphType.scala similarity index 98% rename from s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala rename to s2core/src/main/scala/org/apache/s2graph/core/types/GraphType.scala index c03319d5..acec389b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/GraphType.scala @@ -21,7 +21,7 @@ package org.apache.s2graph.core.types import org.apache.hadoop.hbase.util.Bytes -object HBaseType { +object GraphType { val VERSION4 = "v4" val VERSION3 = "v3" val VERSION2 = "v2" @@ -38,7 +38,7 @@ object HBaseType { object HBaseDeserializable { - import HBaseType._ + import GraphType._ // 6 bits is used for index sequence so total index per label is limited to 2^6 def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = { @@ -151,7 +151,7 @@ trait HBaseSerializable { trait HBaseDeserializable { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, @@ -167,7 +167,7 @@ trait HBaseDeserializable { trait HBaseDeserializableWithIsVertexId { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala index 09d065f5..4e44d27b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala @@ -22,7 +22,7 @@ package org.apache.s2graph.core.types import org.apache.hadoop.hbase.util._ object InnerVal extends HBaseDeserializableWithIsVertexId { - import HBaseType._ + import GraphType._ val order = Order.DESCENDING val stringLenOffset = 7.toByte @@ -234,7 +234,7 @@ trait InnerValLike extends HBaseSerializable { } object InnerValLikeWithTs extends HBaseDeserializable { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala index a923e6f3..8d2accfe 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala @@ -24,7 +24,7 @@ import org.apache.s2graph.core.GraphUtil object LabelWithDirection { - import HBaseType._ + import GraphType._ def apply(compositeInt: Int): LabelWithDirection = { // logger.debug(s"CompositeInt: $compositeInt") @@ -50,7 +50,7 @@ object LabelWithDirection { case class LabelWithDirection(labelId: Int, dir: Int) extends HBaseSerializable { - import HBaseType._ + import GraphType._ assert(dir < (1 << bitsForDir)) assert(labelId < (Int.MaxValue >> bitsForDir)) diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala index 9fe57ea8..4d2d1fc9 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala @@ -21,10 +21,10 @@ package org.apache.s2graph.core.types import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.types.HBaseType._ +import org.apache.s2graph.core.types.GraphType._ object VertexId extends HBaseDeserializable { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, @@ -96,7 +96,7 @@ class VertexId protected (val colId: Int, val innerId: InnerValLike) extends HBa } object SourceVertexId extends HBaseDeserializable { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, @@ -117,7 +117,7 @@ case class SourceVertexId(override val colId: Int, } object TargetVertexId extends HBaseDeserializable { - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, @@ -138,7 +138,7 @@ case class TargetVertexId(override val colId: Int, object SourceAndTargetVertexIdPair extends HBaseDeserializable { val delimiter = ":" - import HBaseType._ + import GraphType._ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala index 361b9cff..5eb50a8a 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala @@ -22,10 +22,10 @@ package org.apache.s2graph.core.types.v1 import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.GraphExceptions import org.apache.s2graph.core.GraphExceptions.IllegalDataTypeException -import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, HBaseType, InnerValLike} +import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, GraphType, InnerValLike} object InnerVal extends HBaseDeserializableWithIsVertexId { - import HBaseType._ + import GraphType._ // val defaultVal = new InnerVal(None, None, None) val stringLenOffset = 7.toByte val maxStringLen = Byte.MaxValue - stringLenOffset diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala index 71106813..1be928a6 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala @@ -21,11 +21,11 @@ package org.apache.s2graph.core.types.v2 import org.apache.hadoop.hbase.util._ import org.apache.s2graph.core.types -import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, HBaseType, InnerValLike} +import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, GraphType, InnerValLike} object InnerVal extends HBaseDeserializableWithIsVertexId { - import HBaseType._ + import GraphType._ val order = Order.DESCENDING diff --git a/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala new file mode 100644 index 00000000..1450b7cf --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala @@ -0,0 +1,14 @@ +package org.apache.s2graph.core + +import org.scalatest.Tag + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/23. + */ +object CommonTest extends Tag("CommonTest") +object V1Test extends Tag("V1Test") +object V2Test extends Tag("V2Test") +object V3Test extends Tag("V3Test") +object V4Test extends Tag("V4Test") +object HBaseTest extends Tag("HBaseTest") +object RedisTest extends Tag("RedisTest") diff --git a/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala index a018c015..d0727cf0 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala @@ -25,530 +25,171 @@ import org.apache.s2graph.core.utils.logger import org.scalatest.FunSuite class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels { - initTests() - test("toLogString") { - val testLabelName = labelNameV2 - val bulkQueries = List( - ("1445240543366", "update", "{\"is_blocked\":true}"), - ("1445240543362", "insert", "{\"is_hidden\":false}"), - ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), - ("1445240543363", "delete", "{}"), - ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) - - val (srcId, tgtId, labelName) = ("1", "2", testLabelName) - - val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { - Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString - }).mkString("\n") - - val expected = Seq( - Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"), - Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"), - Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"), - Seq("1445240543363", "delete", "e", "1", "2", testLabelName), - Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}") - ).map(_.mkString("\t")).mkString("\n") - - assert(bulkEdge === expected) - } - - test("buildOperation") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - - val newPropsWithTs = Map( - timestampProp, - 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1) - ) - - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isDefined) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: snapshotEdge: None with newProps") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - - val newPropsWithTs = Map( - timestampProp, - 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1) - ) - - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isDefined) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - - val newPropsWithTs = propsWithTs - - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isEmpty) - assert(edgeMutate.edgesToInsert.isEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: All props older than snapshotEdge's LastDeletedAt") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - val oldPropsWithTs = Map( - timestampProp, - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val propsWithTs = Map( - timestampProp, - 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 2), - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val snapshotEdge = - Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) - - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.nonEmpty) - assert(edgeMutate.edgesToInsert.isEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + initTests(ver) + + + val label = labelName(ver) + val dirLabel = labelWithDir(ver) +// test(s"toLogString $ver", tag) { +// val bulkQueries = List( +// ("1445240543366", "update", "{\"is_blocked\":true}"), +// ("1445240543362", "insert", "{\"is_hidden\":false}"), +// ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), +// ("1445240543363", "delete", "{}"), +// ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) +// +// val (srcId, tgtId, labelName) = ("1", "2", label) +// +// val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { +// Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString +// }).mkString("\n") +// +// val expected = Seq( +// Seq("1445240543366", "update", "e", "1", "2", label, "{\"is_blocked\":true}"), +// Seq("1445240543362", "insert", "e", "1", "2", label, "{\"is_hidden\":false}"), +// Seq("1445240543364", "insert", "e", "1", "2", label, "{\"is_hidden\":false,\"weight\":10}"), +// Seq("1445240543363", "delete", "e", "1", "2", label), +// Seq("1445240543365", "update", "e", "1", "2", label, "{\"time\":1,\"weight\":-10}") +// ).map(_.mkString("\t")).mkString("\n") +// +// assert(bulkEdge === expected) +// } - test("buildMutation: All props newer than snapshotEdge's LastDeletedAt") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex + test(s"buildOperation $ver", tag) { + val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) + val srcVertex = Vertex(vertexId) + val tgtVertex = srcVertex - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - val oldPropsWithTs = Map( - timestampProp, - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) + val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) - val propsWithTs = Map( - timestampProp, - 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 4), - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) + val snapshotEdge = None + val propsWithTs = Map(timestampProp) + val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) + val newVersion = 0L - val snapshotEdge = - Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) + val newPropsWithTs = Map( + timestampProp, + 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, ver), 1) + ) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) - logger.info(edgeMutate.toLogString) + val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) + logger.info(edgeMutate.toLogString) - assert(edgeMutate.newSnapshotEdge.nonEmpty) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } -} + assert(edgeMutate.newSnapshotEdge.isDefined) + assert(edgeMutate.edgesToInsert.nonEmpty) + assert(edgeMutate.edgesToDelete.isEmpty) + } -//import com.kakao.s2graph.core.types._ -//import org.hbase.async.PutRequest -//import org.scalatest.{FunSuite, Matchers} -// -///** -// * Created by shon on 5/29/15. -// */ -//class EdgeTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels { -// -// -// import HBaseType.{VERSION1, VERSION2} +// test(s"buildMutation: snapshotEdge: None with newProps $ver", tag) { +// val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) +// val srcVertex = Vertex(vertexId) +// val tgtVertex = srcVertex // +// val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) // -// def srcVertex(innerVal: InnerValLike)(version: String) = { -// val colId = if (version == VERSION1) column.id.get else columnV2.id.get -// Vertex(SourceVertexId(colId, innerVal), ts) -// } +// val snapshotEdge = None +// val propsWithTs = Map(timestampProp) +// val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) +// val newVersion = 0L // -// def tgtVertex(innerVal: InnerValLike)(version: String) = { -// val colId = if (version == VERSION1) column.id.get else columnV2.id.get -// Vertex(TargetVertexId(colId, innerVal), ts) -// } +// val newPropsWithTs = Map( +// timestampProp, +// 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, ver), 1) +// ) // +// val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) +// logger.info(edgeMutate.toLogString) // -// def testEdges(version: String) = { -// val innerVals = if (version == VERSION1) intInnerVals else intInnerValsV2 -// val tgtV = tgtVertex(innerVals.head)(version) -// innerVals.tail.map { intInnerVal => -// val labelWithDirection = if (version == VERSION1) labelWithDir else labelWithDirV2 -// val idxPropsList = if (version == VERSION1) idxPropsLs else idxPropsLsV2 -// idxPropsList.map { idxProps => -// val currentTs = idxProps.toMap.get(0.toByte).get.toString.toLong -// val idxPropsWithTs = idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, currentTs) } -// -// Edge(srcVertex(intInnerVal)(version), tgtV, labelWithDirection, op, currentTs, currentTs, idxPropsWithTs.toMap) -// } +// assert(edgeMutate.newSnapshotEdge.isDefined) +// assert(edgeMutate.edgesToInsert.nonEmpty) +// assert(edgeMutate.edgesToDelete.isEmpty) // } -// } -// -// def testPropsUpdate(oldProps: Map[Byte, InnerValLikeWithTs], -// newProps: Map[Byte, InnerValLikeWithTs], -// expected: Map[Byte, Any], -// expectedShouldUpdate: Boolean) -// (f: PropsPairWithTs => (Map[Byte, InnerValLikeWithTs], Boolean))(version: String) = { -// -// val timestamp = newProps.toList.head._2.ts -// val (updated, shouldUpdate) = f((oldProps, newProps, timestamp, version)) -// val rets = for { -// (k, v) <- expected -// } yield { -// v match { -// case v: String => -// v match { -// case "left" => updated.get(k).isDefined && updated(k) == oldProps(k) -// case "right" => updated.get(k).isDefined && updated(k) == newProps(k) -// case "none" => updated.get(k).isEmpty -// } -// case value: InnerValLikeWithTs => updated.get(k).get == value -// case _ => throw new RuntimeException(s"not supported keyword: $v") -// } -// } -// println(rets) -// rets.forall(x => x) && shouldUpdate == expectedShouldUpdate -// } -// -// def testEdgeWithIndex(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = { -// val rets = for { -// edgeForSameTgtVertex <- edges -// } yield { -// val head = edgeForSameTgtVertex.head -// val start = head -// var prev = head -// val rets = for { -// edge <- edgeForSameTgtVertex.tail -// } yield { -// println(s"prevEdge: $prev") -// println(s"currentEdge: $edge") -// val prevEdgeWithIndex = edge.edgesWithIndex -// val edgesWithIndex = edge.edgesWithIndex -// -// /** test encodeing decoding */ -// for { -// edgeWithIndex <- edge.edgesWithIndex -// put <- edgeWithIndex.buildPutsAsync() -// kv <- putToKeyValues(put.asInstanceOf[PutRequest]) -// } { -// val decoded = Edge.toEdge(kv, queryParam, None, Seq()) -// val comp = decoded.isDefined && decoded.get == edge -// println(s"${decoded.get}") -// println(s"$edge") -// println(s"${decoded.get == edge}") -// comp shouldBe true -// -// /** test order -// * same source, target vertex. same indexProps keys. -// * only difference is indexProps values so comparing qualifier is good enough -// * */ -// for { -// prevEdgeWithIndex <- prev.edgesWithIndex -// } { -// println(edgeWithIndex.qualifier) -// println(prevEdgeWithIndex.qualifier) -// println(edgeWithIndex.qualifier.bytes.toList) -// println(prevEdgeWithIndex.qualifier.bytes.toList) -// /** since index of this test label only use 0, 1 as indexProps -// * if 0, 1 is not different then qualifier bytes should be same -// * */ -// val comp = lessThanEqual(edgeWithIndex.qualifier.bytes, prevEdgeWithIndex.qualifier.bytes) -// comp shouldBe true -// } -// } -// prev = edge -// } -// } -// } -// -// def testInvertedEdge(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = { -// val rets = for { -// edgeForSameTgtVertex <- edges -// } yield { -// val head = edgeForSameTgtVertex.head -// val start = head -// var prev = head -// val rets = for { -// edge <- edgeForSameTgtVertex.tail -// } yield { -// println(s"prevEdge: $prev") -// println(s"currentEdge: $edge") -// val prevEdgeWithIndexInverted = prev.toSnapshotEdge -// val edgeWithInvertedIndex = edge.toSnapshotEdge -// /** test encode decoding */ -// -// val put = edgeWithInvertedIndex.buildPutAsync() -// for { -// kv <- putToKeyValues(put) -// } yield { -// val decoded = Edge.toSnapshotEdge(kv, queryParam, None, isInnerCall = false, Seq()) -// val comp = decoded.isDefined && decoded.get == edge -// println(s"${decoded.get}") -// println(s"$edge") -// println(s"${decoded.get == edge}") -// comp shouldBe true -// -// /** no need to test ordering because qualifier only use targetVertexId */ -// } -// prev = edge -// } -// } -// } -// -// test("insert for edgesWithIndex version 2") { -// val version = VERSION2 -// testEdgeWithIndex(testEdges(version))(queryParamV2) -// } -// test("insert for edgesWithIndex version 1") { -// val version = VERSION1 -// testEdgeWithIndex(testEdges(version))(queryParam) -// } -// -// test("insert for edgeWithInvertedIndex version 1") { -// val version = VERSION1 -// testInvertedEdge(testEdges(version))(queryParam) -// } -// -// test("insert for edgeWithInvertedIndex version 2") { -// val version = VERSION2 -// testInvertedEdge(testEdges(version))(queryParamV2) -// } // +// test(s"buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests $ver", tag) { +// val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) +// val srcVertex = Vertex(vertexId) +// val tgtVertex = srcVertex // -// // /** test cases for each operation */ +// val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) // -// def oldProps(timestamp: Long, version: String) = { -// Map( -// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp - 2, timestamp - 2, version), -// 1.toByte -> InnerValLikeWithTs.withLong(0L, timestamp, version), -// 2.toByte -> InnerValLikeWithTs.withLong(1L, timestamp - 1, version), -// 4.toByte -> InnerValLikeWithTs.withStr("old", timestamp - 1, version) -// ) -// } +// val snapshotEdge = None +// val propsWithTs = Map(timestampProp) +// val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) +// val newVersion = 0L // -// def newProps(timestamp: Long, version: String) = { -// Map( -// 2.toByte -> InnerValLikeWithTs.withLong(-10L, timestamp, version), -// 3.toByte -> InnerValLikeWithTs.withLong(20L, timestamp, version) -// ) -// } +// val newPropsWithTs = propsWithTs // -// def deleteProps(timestamp: Long, version: String) = Map( -// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp, timestamp, version) -// ) +// val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) +// logger.info(edgeMutate.toLogString) // -// /** upsert */ -// test("Edge.buildUpsert") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "none", -// 2.toByte -> "right", -// 3.toByte -> "right", -// 4.toByte -> "none") -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true -// } -// test("Edge.buildUpsert shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 3.toByte -> "none", -// 4.toByte -> "left") -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true -// } -// -// /** update */ -// test("Edge.buildUpdate") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "right", -// 3.toByte -> "right", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true -// } -// test("Edge.buildUpdate shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 3.toByte -> "none", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true -// } -// -// /** delete */ -// test("Edge.buildDelete") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = deleteProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "right", -// 1.toByte -> "none", -// 2.toByte -> "none", -// 4.toByte -> "none" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true -// } -// test("Edge.buildDelete shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = deleteProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true -// } -// -// /** increment */ -// test("Edge.buildIncrement") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> InnerValLikeWithTs.withLong(-9L, ts - 1, VERSION2), -// 3.toByte -> "right" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true -// } -// test("Edge.buildIncrement shouldRepalce false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true -// } -// -// test("Edge`s srcVertex") { -// -// val version = VERSION2 -// val srcId = InnerVal.withLong(10, version) -// val tgtId = InnerVal.withStr("abc", version) -// val srcColumn = columnV2 -// val tgtColumn = tgtColumnV2 -// val srcVertexId = VertexId(srcColumn.id.get, srcId) -// val tgtVertexId = VertexId(tgtColumn.id.get, tgtId) -// -// val srcVertex = Vertex(srcVertexId) -// val tgtVertex = Vertex(tgtVertexId) -// -// val labelId = undirectedLabelV2.id.get -// -// val outDir = LabelWithDirection(labelId, GraphUtil.directions("out")) -// val inDir = LabelWithDirection(labelId, GraphUtil.directions("in")) -// val bothDir = LabelWithDirection(labelId, GraphUtil.directions("undirected")) -// -// val op = GraphUtil.operations("insert") -// -// -// val bothEdge = Edge(srcVertex, tgtVertex, bothDir) -// println(s"edge: $bothEdge") -// bothEdge.relatedEdges.foreach { edge => -// println(edge) +// assert(edgeMutate.newSnapshotEdge.isEmpty) +// assert(edgeMutate.edgesToInsert.isEmpty) +// assert(edgeMutate.edgesToDelete.isEmpty) // } // -// } -// test("edge buildIncrementBulk") { -// -// /** -// * 172567371 List(97, 74, 2, 117, -74, -44, -76, 0, 0, 4, 8, 2) -//169116518 List(68, -110, 2, 117, -21, 124, -103, 0, 0, 4, 9, 2) -//11646834 List(17, 33, 2, 127, 78, 72, -115, 0, 0, 4, 9, 2) -//148171217 List(62, 54, 2, 119, 43, 22, 46, 0, 0, 4, 9, 2) -//116315188 List(41, 86, 2, 121, 17, 43, -53, 0, 0, 4, 9, 2) -//180667876 List(48, -82, 2, 117, 59, 58, 27, 0, 0, 4, 8, 2) -//4594410 List(82, 29, 2, 127, -71, -27, 21, 0, 0, 4, 8, 2) -//151435444 List(1, 105, 2, 118, -7, 71, 75, 0, 0, 4, 8, 2) -//168460895 List(67, -35, 2, 117, -11, 125, -96, 0, 0, 4, 9, 2) -//7941614 List(115, 67, 2, 127, -122, -46, 17, 0, 0, 4, 8, 2) -//171169732 List(61, -42, 2, 117, -52, 40, 59, 0, 0, 4, 9, 2) -//174381375 List(91, 2, 2, 117, -101, 38, -64, 0, 0, 4, 9, 2) -//12754019 List(9, -80, 2, 127, 61, 99, -100, 0, 0, 4, 9, 2) -//175518092 List(111, 32, 2, 117, -119, -50, 115, 0, 0, 4, 8, 2) -//174748531 List(28, -81, 2, 117, -107, -116, -116, 0, 0, 4, 8, 2) +// test(s"buildMutation: All props older than snapshotEdge's LastDeletedAt $ver", tag) { +// val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) +// val srcVertex = Vertex(vertexId) +// val tgtVertex = srcVertex +// +// val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) +// val oldPropsWithTs = Map( +// timestampProp, +// LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) +// ) +// +// val propsWithTs = Map( +// timestampProp, +// 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 2), +// LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) +// ) +// +// val snapshotEdge = +// Option(Edge(srcVertex, tgtVertex, dirLabel, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) +// +// val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) +// val newVersion = 0L +// val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) +// logger.info(edgeMutate.toLogString) +// +// assert(edgeMutate.newSnapshotEdge.nonEmpty) +// assert(edgeMutate.edgesToInsert.isEmpty) +// assert(edgeMutate.edgesToDelete.isEmpty) +// } // -// */ -// // val incrementsOpt = Edge.buildIncrementDegreeBulk("169116518", "talk_friend_long_term_agg_test", "out", 10) -// // -// // for { -// // increments <- incrementsOpt -// // increment <- increments -// // (cf, qs) <- increment.getFamilyMapOfLongs -// // (q, v) <- qs -// // } { -// // println(increment.getRow.toList) -// // println(q.toList) -// // println(v) -// // } -// //List(68, -110, -29, -4, 116, -24, 124, -37, 0, 0, 52, -44, 2) -// } -//} +// test(s"buildMutation: All props newer than snapshotEdge's LastDeletedAt $ver", tag) { +// val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) +// val srcVertex = Vertex(vertexId) +// val tgtVertex = srcVertex +// +// val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) +// val oldPropsWithTs = Map( +// timestampProp, +// LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) +// ) +// +// val propsWithTs = Map( +// timestampProp, +// 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 4), +// LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) +// ) +// +// val snapshotEdge = +// Option(Edge(srcVertex, tgtVertex, dirLabel, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) +// +// val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) +// val newVersion = 0L +// val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) +// logger.info(edgeMutate.toLogString) +// +// assert(edgeMutate.newSnapshotEdge.nonEmpty) +// assert(edgeMutate.edgesToInsert.nonEmpty) +// assert(edgeMutate.edgesToDelete.isEmpty) +// } + } +} + diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala index 3911e0d2..c13e3bc6 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,158 +26,188 @@ class CrudTest extends IntegrateCommon { import CrudHelper._ import TestUtil._ - test("test CRUD") { - var tcNum = 0 - var tcString = "" - var bulkQueries = List.empty[(Long, String, String)] - var expected = Map.empty[String, String] - - val curTime = System.currentTimeMillis - val t1 = curTime + 0 - val t2 = curTime + 1 - val t3 = curTime + 2 - val t4 = curTime + 3 - val t5 = curTime + 4 - - val tcRunner = new CrudTestRunner() - tcNum = 1 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 2 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 3 - tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 4 - tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 5 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" - bulkQueries = List( - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 6 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " - bulkQueries = List( - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 7 - tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 8 - tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 9 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " - bulkQueries = List( - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 10 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" - bulkQueries = List( - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 11 - tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 12 - tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 13 - tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " - bulkQueries = List( - (t5, "update", "{\"is_blocked\": true}"), - (t1, "insert", "{\"is_hidden\": false}"), - (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), - (t2, "delete", ""), - (t4, "update", "{\"time\": 1, \"weight\": -10}")) - expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - } + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + + test(s"test CRUD $ver", tag) { + val tcRunner = new CrudTestRunner() + tcRunner.test(ver) + } +// val label = getLabelName(ver) +// test(s"interval $ver", tag) { +// +// var edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index +// (edges \ "size").toString should be("1") +// +// edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index +// (edges \ "size").toString should be("2") +// +// edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 11)) // test interval on weight index +// (edges \ "size").toString should be("1") +// +// edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 20)) // test interval on weight index +// (edges \ "size").toString should be("2") +// } + } object CrudHelper { class CrudTestRunner { var seed = 0 - def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { + def test(ver: String) = { + var tcNum = 0 + var tcString = "" + var bulkQueries = List.empty[(Long, String, String)] + var expected = Map.empty[String, String] + + val curTime = System.currentTimeMillis + val t1 = curTime + 0 + val t2 = curTime + 1 + val t3 = curTime + 2 + val t4 = curTime + 3 + val t5 = curTime + 4 + + tcNum = 1 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 2 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 3 + tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 4 + tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 5 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" + bulkQueries = List( + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 6 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " + bulkQueries = List( + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 7 + tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 8 + tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 9 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " + bulkQueries = List( + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 10 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" + bulkQueries = List( + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 11 + tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 12 + tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 13 + tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " + bulkQueries = List( + (t5, "update", "{\"is_blocked\": true}"), + (t1, "insert", "{\"is_hidden\": false}"), + (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), + (t2, "delete", ""), + (t4, "update", "{\"time\": 1, \"weight\": -10}")) + expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") + + run(ver, tcNum, tcString, bulkQueries, expected) + } + + def run(ver: String, tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { + val labelName = getLabelName(ver) + for { - labelName <- List(testLabelName, testLabelName2) i <- 0 until NumOfEachTest } { seed += 1 @@ -188,9 +218,9 @@ class CrudTest extends IntegrateCommon { /** insert edges */ println(s"---- TC${tcNum}_init ----") - val bulkEdges = (for ((ts, op, props) <- opWithProps) yield { + val bulkEdges = for ((ts, op, props) <- opWithProps) yield { TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props) - }) + } TestUtil.insertEdgesSync(bulkEdges: _*) @@ -204,15 +234,15 @@ class CrudTest extends IntegrateCommon { case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId) } - val qId = if (labelName == testLabelName) id else "\"" + id + "\"" + val qId = if (labelName == testLabelNameV3 || labelName == testLabelNameV4) id else "\"" + id + "\"" val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL) val jsResult = TestUtil.getEdgesSync(query) val results = jsResult \ "results" - val deegrees = (jsResult \ "degrees").as[List[JsObject]] + val degrees = (jsResult \ "degrees").as[List[JsObject]] val propsLs = (results \\ "props").seq - (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1) + (degrees.head \ LabelMeta.degree.name).as[Int] should be(1) val from = (results \\ "from").seq.last.toString.replaceAll("\"", "") val to = (results \\ "to").seq.last.toString.replaceAll("\"", "") diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala index 225d3963..04467aa7 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,24 +20,21 @@ package org.apache.s2graph.core.Integrate import com.typesafe.config._ +import org.apache.s2graph.core._ import org.apache.s2graph.core.mysqls.Label import org.apache.s2graph.core.rest.{RequestParser, RestHandler} import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.core.{Graph, GraphUtil, Management, PostProcess} import org.scalatest._ import play.api.libs.json.{JsValue, Json} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} +import scala.util.Random -trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { +trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll with TestCommon{ import TestUtil._ - var graph: Graph = _ - var parser: RequestParser = _ - var management: Management = _ - var config: Config = _ override def beforeAll = { config = ConfigFactory.load() @@ -51,6 +48,9 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { graph.shutdown() } + def getLabelName(ver: String, consistency: String = "strong") = s"s2graph_label_test_${ver}_${consistency}" + def getLabelName2(ver: String, consistency: String = "strong") = s"s2graph_label_test_${ver}_${consistency}_2" + /** * Make Service, Label, Vertex for integrate test */ @@ -67,14 +67,16 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) println(s">> Service created : $createService, $tryRes") - val labelNames = Map(testLabelName -> testLabelNameCreate, - testLabelName2 -> testLabelName2Create, - testLabelNameV1 -> testLabelNameV1Create, - testLabelNameWeak -> testLabelNameWeakCreate) + // val labelNames = Map(testLabelNameV4 -> testLabelNameCreate(ver), + // testLabelNameV3 -> testLabelNameCreate(ver), + // testLabelNameV1 -> testLabelNameCreate(ver), + // testLabelNameWeak -> testLabelNameWeakCreate(ver)) - for { - (labelName, create) <- labelNames - } { + // Create test labels by versions + consistency. + for (n <- versions; consistency <- Seq("strong", "weak")) yield { + val ver = s"v$n" + val labelName = getLabelName(ver, consistency) + val create = testLabelCreate(ver, consistency) Management.deleteLabel(labelName) Label.findByName(labelName, useCache = false) match { case None => @@ -117,6 +119,40 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { // s // } + def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { + Json.parse( + s""" + |[ + |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}]} + |] + """.stripMargin) + } + + def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { + ids.map { id => + Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) + } + } + + val vertexPropsKeys = List( + ("age", "int") + ) + + def randomProps() = { + (for { + (propKey, propType) <- vertexPropsKeys + } yield { + propKey -> Random.nextInt(100) + }).toMap + } + + def getVerticesSync(queryJson: JsValue): JsValue = { + val restHandler = new RestHandler(graph) + logger.info(Json.prettyPrint(queryJson)) + val f = restHandler.getVertices(queryJson) + Await.result(f, HttpRequestWaitingTime) + } + def deleteAllSync(jsValue: JsValue) = { val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json => val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json) @@ -128,12 +164,33 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { Await.result(future, HttpRequestWaitingTime) } + def getEdgesSync(queryJson: JsValue): JsValue = { logger.info(Json.prettyPrint(queryJson)) val restHandler = new RestHandler(graph) Await.result(restHandler.getEdgesAsync(queryJson)(PostProcess.toSimpleVertexArrJson), HttpRequestWaitingTime) } + def checkEdgesSync(checkEdgeJson: JsValue): JsValue = { + logger.info(Json.prettyPrint(checkEdgeJson)) + + val ret = parser.toCheckEdgeParam(checkEdgeJson) match { + case (e, _) => graph.checkEdges(e) + } + val result = Await.result(ret, HttpRequestWaitingTime) + val jsResult = PostProcess.toSimpleVertexArrJson(result) + + logger.info(jsResult.toString) + jsResult + } + + def mutateEdgesSync(bulkEdges: String*) = { + val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true) + val jsResult = Await.result(req, HttpRequestWaitingTime) + + jsResult + } + def insertEdgesSync(bulkEdges: String*) = { val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true) val jsResult = Await.result(req, HttpRequestWaitingTime) @@ -150,10 +207,14 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { // common tables val testServiceName = "s2graph" - val testLabelName = "s2graph_label_test" - val testLabelName2 = "s2graph_label_test_2" - val testLabelNameV1 = "s2graph_label_test_v1" - val testLabelNameWeak = "s2graph_label_test_weak" + val testLabelNameV4 = "s2graph_label_test_v4_strong" + val testLabelNameV3 = "s2graph_label_test_v3_strong" + val testLabelNameV2 = "s2graph_label_test_v2_strong" + val testLabelNameV1 = "s2graph_label_test_v1_strong" + val testLabelNameWeakV4 = "s2graph_label_test_v4_weak" + val testLabelNameWeakV3 = "s2graph_label_test_v3_weak" + val testLabelNameWeakV2 = "s2graph_label_test_v2_weak" + val testLabelNameWeakV1 = "s2graph_label_test_v1_weak" val testColumnName = "user_id_test" val testColumnType = "long" val testTgtColumnName = "item_id_test" @@ -167,14 +228,15 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { val createService = s"""{"serviceName" : "$testServiceName"}""" - val testLabelNameCreate = + def testLabelCreate(ver: String, consistency: String = "strong") = { + val label = getLabelName(ver, consistency) s""" { - "label": "$testLabelName", + "label": "$label", "srcServiceName": "$testServiceName", "srcColumnName": "$testColumnName", "srcColumnType": "long", - "tgtServiceName": "$testServiceName", + "tgtServiceName": "$testServiceName", "tgtColumnName": "$testColumnName", "tgtColumnType": "long", "indices": [ @@ -203,23 +265,27 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { "defaultValue": false } ], - "consistencyLevel": "strong", - "schemaVersion": "v4", - "compressionAlgorithm": "gz", - "hTableName": "$testHTableName" + "consistencyLevel": "$consistency", + "schemaVersion": "$ver", + "compressionAlgorithm": "gz" }""" + } - val testLabelName2Create = + def testLabel2Create(ver: String, consistency: String = "strong") = { + val label = getLabelName2(ver, consistency) s""" { - "label": "$testLabelName2", + "label": "$label", "srcServiceName": "$testServiceName", "srcColumnName": "$testColumnName", "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], + "tgtServiceName": "$testServiceName", + "tgtColumnName": "$testColumnName", + "tgtColumnType": "long", + "indices": [ + {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, + {"name": "$index2", "propNames": ["_timestamp"]} + ], "props": [ { "name": "time", @@ -242,87 +308,295 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { "defaultValue": false } ], - "consistencyLevel": "strong", - "isDirected": false, - "schemaVersion": "v3", + "consistencyLevel": "$consistency", + "schemaVersion": "$ver", "compressionAlgorithm": "gz" }""" + } - val testLabelNameV1Create = + def querySingle(id: Int, label: String, offset: Int = 0, limit: Int = 100) = Json.parse( s""" - { - "label": "$testLabelNameV1", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "${testTgtColumnName}_v1", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$label", + "direction": "out", + "offset": $offset, + "limit": $limit + } + ]] + } + """) + + def queryWithInterval(label: String, id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$label", + "index": "$index", + "interval": { + "from": [ { "$prop": $fromVal } ], + "to": [ { "$prop": $toVal } ] + } + } + ]] + } + """) + + def queryWhere(id: Int, label: String, where: String) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "limit": 100, + "where": "${where}" + } + ]] + }""") + + def queryExclude(id: Int, label: String) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "limit": 2 + }, + { + "label": "${label}", + "direction": "in", + "offset": 0, + "limit": 2, + "exclude": true + } + ]] + }""") + + def queryGroupBy(id: Int, label: String, props: Seq[String]): JsValue = { + Json.obj( + "groupBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label + ) + ) + ) + ) + ) } - ], - "consistencyLevel": "strong", - "isDirected": true, - "schemaVersion": "v1", - "compressionAlgorithm": "gz" - }""" - val testLabelNameWeakCreate = + def queryTransform(id: Int, label: String, transforms: String) = Json.parse( s""" - { - "label": "$testLabelNameWeak", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "transform": $transforms + } + ]] + }""") + + def queryIndex(ids: Seq[Int], label: String, indexName: String) = { + val $from = Json.arr( + Json.obj("serviceName" -> testServiceName, + "columnName" -> testColumnName, + "ids" -> ids)) + + val $step = Json.arr(Json.obj("label" -> label, "index" -> indexName)) + val $steps = Json.arr(Json.obj("step" -> $step)) + + val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps) + js } - ], - "consistencyLevel": "weak", - "isDirected": true, - "compressionAlgorithm": "gz" - }""" + + def queryParents(id: Long) = Json.parse( + s""" + { + "returnTree": true, + "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$testLabelNameV3", + "direction": "out", + "offset": 0, + "limit": 2 + } + ],[{ + "label": "$testLabelNameV3", + "direction": "in", + "offset": 0, + "limit": -1 + } + ]] + }""".stripMargin) + + def querySingleWithTo(id: Int, label: String, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": $offset, + "limit": $limit, + "_to": $to + } + ]] + } + """) + + def queryScore(id: Int, label: String, scoring: Map[String, Int]): JsValue = Json.obj( + "srcVertices" -> Json.arr( + Json.obj( + "serviceName" -> testServiceName, + "columnName" -> testColumnName, + "id" -> id + ) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label, + "scoring" -> scoring + ) + ) + ) + ) + ) + + def queryOrderBy(id: Int, label: String, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj( + "orderBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label, + "scoring" -> scoring + ) + ) + ) + ) + ) + + def queryWithSampling(id: Int, label: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + } + ] + }""") + + def twoStepQueryWithSampling(id: Int, label: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + }, + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + } + ] + }""") + + def twoQueryWithSampling(id: Int, label: String, label2: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 50, + "sample": $sample + }, + { + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 50 + }] + } + ] + }""") } + } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala index 8bdc99b4..9000e5b3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala @@ -1,27 +1,8 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.s2graph.core.Integrate import org.apache.s2graph.core.utils.logger import org.scalatest.BeforeAndAfterEach -import play.api.libs.json.{JsNumber, JsValue, Json} +import play.api.libs.json._ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { @@ -32,1064 +13,232 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { val weight = "weight" val is_hidden = "is_hidden" - test("interval") { - def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "index": "$index", - "interval": { - "from": [ { "$prop": $fromVal } ], - "to": [ { "$prop": $toVal } ] - } - } - ]] - } - """) - - var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index - (edges \ "size").toString should be("1") + versions map { n => - edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index - (edges \ "size").toString should be("2") + val ver = s"v$n" + val label = getLabelName(ver) + val label2 = getLabelName2(ver) + val tag = getTag(ver) - edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index - (edges \ "size").toString should be("1") + test(s"interval $ver", tag) { - edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index - (edges \ "size").toString should be("2") - } + var edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index + (edges \ "size").toString should be("1") - test("get edge with where condition") { - def queryWhere(id: Int, where: String) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "where": "${where}" - } - ]] - }""") + edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index + (edges \ "size").toString should be("2") - var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)")) - (result \ "results").as[List[JsValue]].size should be(1) + edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 11)) // test interval on weight index + (edges \ "size").toString should be("1") - result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)")) - (result \ "results").as[List[JsValue]].size should be(1) + edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 20)) // test interval on weight index + (edges \ "size").toString should be("2") + } - result = getEdgesSync(queryWhere(0, "_from=0")) - (result \ "results").as[List[JsValue]].size should be(2) + test(s"get edge with where condition $ver", tag) { - result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)")) - (result \ "results").as[List[JsValue]].size should be(2) + var result = getEdgesSync(queryWhere(0, label, "is_hidden=false and _from in (-1, 0)")) + (result \ "results").as[List[JsValue]].size should be(1) - result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)")) - (result \ "results").as[List[JsValue]].size should be(2) - } + result = getEdgesSync(queryWhere(0, label, "is_hidden=true and _to in (1)")) + (result \ "results").as[List[JsValue]].size should be(1) - test("get edge exclude") { - def queryExclude(id: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 2 - }, - { - "label": "${testLabelName}", - "direction": "in", - "offset": 0, - "limit": 2, - "exclude": true - } - ]] - }""") + result = getEdgesSync(queryWhere(0, label, "_from=0")) + (result \ "results").as[List[JsValue]].size should be(2) - val result = getEdgesSync(queryExclude(0)) - (result \ "results").as[List[JsValue]].size should be(1) - } + result = getEdgesSync(queryWhere(2, label, "_from=2 or weight in (-1)")) + (result \ "results").as[List[JsValue]].size should be(2) - test("get edge groupBy property") { - def queryGroupBy(id: Int, props: Seq[String]): JsValue = { - Json.obj( - "groupBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName - ) - ) - ) - ) - ) + result = getEdgesSync(queryWhere(2, label, "_from=2 and weight in (10, 20)")) + (result \ "results").as[List[JsValue]].size should be(2) } - val result = getEdgesSync(queryGroupBy(0, Seq("weight"))) - (result \ "size").as[Int] should be(2) - val weights = (result \ "results" \\ "groupBy").map { js => - (js \ "weight").as[Int] + test(s"get edge exclude $ver", tag) { + val result = getEdgesSync(queryExclude(0, label)) + (result \ "results").as[List[JsValue]].size should be(1) } - weights should contain(30) - weights should contain(40) - - weights should not contain (10) - } - - test("edge transform") { - def queryTransform(id: Int, transforms: String) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "transform": $transforms - } - ]] - }""") - - var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]")) - (result \ "results").as[List[JsValue]].size should be(2) - result = getEdgesSync(queryTransform(0, "[[\"weight\"]]")) - (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "weight").map(_.toString).sorted) + test(s"get edge groupBy property $ver", tag) { - result = getEdgesSync(queryTransform(0, "[[\"_from\"]]")) - (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "from").map(_.toString).sorted) - } - - test("index") { - def queryIndex(ids: Seq[Int], indexName: String) = { - val $from = Json.arr( - Json.obj("serviceName" -> testServiceName, - "columnName" -> testColumnName, - "ids" -> ids)) - - val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName)) - val $steps = Json.arr(Json.obj("step" -> $step)) + val result = getEdgesSync(queryGroupBy(0, label, Seq("weight"))) + (result \ "size").as[Int] should be(2) + val weights = (result \\ "groupBy").map { js => + (js \ "weight").as[Int] + } + weights should contain(30) + weights should contain(40) - val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps) - js + weights should not contain (10) } - // weight order - var result = getEdgesSync(queryIndex(Seq(0), "idx_1")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40)) - - // timestamp order - result = getEdgesSync(queryIndex(Seq(0), "idx_2")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30)) - } - - // "checkEdges" in { - // running(FakeApplication()) { - // val json = Json.parse( s""" - // [{"from": 0, "to": 1, "label": "$testLabelName"}, - // {"from": 0, "to": 2, "label": "$testLabelName"}] - // """) - // - // def checkEdges(queryJson: JsValue): JsValue = { - // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get - // contentAsJson(ret) - // } - // - // val res = checkEdges(json) - // val typeRes = res.isInstanceOf[JsArray] - // typeRes must equalTo(true) - // - // val fst = res.as[Seq[JsValue]].head \ "to" - // fst.as[Int] must equalTo(1) - // - // val snd = res.as[Seq[JsValue]].last \ "to" - // snd.as[Int] must equalTo(2) - // } - // } + test(s"edge transform $ver", tag) { + var result = getEdgesSync(queryTransform(0, label, "[[\"_to\"]]")) + (result \ "results").as[List[JsValue]].size should be(2) + result = getEdgesSync(queryTransform(0, label, "[[\"weight\"]]")) + (result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted) - test("duration") { - def queryDuration(ids: Seq[Int], from: Int, to: Int) = { - val $from = Json.arr( - Json.obj("serviceName" -> testServiceName, - "columnName" -> testColumnName, - "ids" -> ids)) - - val $step = Json.arr(Json.obj( - "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100, - "duration" -> Json.obj("from" -> from, "to" -> to))) - - val $steps = Json.arr(Json.obj("step" -> $step)) - - Json.obj("srcVertices" -> $from, "steps" -> $steps) + result = getEdgesSync(queryTransform(0, label, "[[\"_from\"]]")) + val results = (result \ "results").as[JsValue] + (result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted) } - // get all - var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size should be(4) - // inclusive, exclusive - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size should be(3) - - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size should be(1) - - val bulkEdges = Seq( - toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), - toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), - toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)), - toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)) - ) - insertEdgesSync(bulkEdges: _*) - // duration test after udpate - // get all - result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size should be(4) + test(s"index $ver", tag) { + // weight order + var result = getEdgesSync(queryIndex(Seq(0), label, "idx_1")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40)) - // inclusive, exclusive - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size should be(3) - - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size should be(1) - - } + // timestamp order + result = getEdgesSync(queryIndex(Seq(0), label, "idx_2")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30)) + } - test("return tree") { - def queryParents(id: Long) = Json.parse( - s""" - { - "returnTree": true, - "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 2 - } - ],[{ - "label": "$testLabelName", - "direction": "in", - "offset": 0, - "limit": -1 - } - ]] - }""".stripMargin) + test(s"return tree $ver", tag) { + val src = 100 + val tgt = 200 - val src = 100 - val tgt = 200 + insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, label)) - insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName)) + val result = TestUtil.getEdgesSync(queryParents(src)) + val parents = (result \ "results").as[Seq[JsValue]] + val ret = parents.forall { + edge => (edge \ "parents").as[Seq[JsValue]].size == 1 + } - val result = TestUtil.getEdgesSync(queryParents(src)) - val parents = (result \ "results").as[Seq[JsValue]] - val ret = parents.forall { - edge => (edge \ "parents").as[Seq[JsValue]].size == 1 + ret should be(true) } - ret should be(true) - } - -// test("pagination and _to") { -// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( -// s""" -// { "srcVertices": [ -// { "serviceName": "${testServiceName}", -// "columnName": "${testColumnName}", -// "id": ${id} -// }], -// "steps": [ -// [ { -// "label": "${testLabelName}", -// "direction": "out", -// "offset": $offset, -// "limit": $limit, -// "_to": $to -// } -// ]] -// } -// """) +// test(s"pagination and _to $ver", tag) { +// val src = System.currentTimeMillis().toInt // -// val src = System.currentTimeMillis().toInt +// val bulkEdges = Seq( +// toEdge(1001, insert, e, src, 1, label, Json.obj(weight -> 10, is_hidden -> true)), +// toEdge(2002, insert, e, src, 2, label, Json.obj(weight -> 20, is_hidden -> false)), +// toEdge(3003, insert, e, src, 3, label, Json.obj(weight -> 30)), +// toEdge(4004, insert, e, src, 4, label, Json.obj(weight -> 40)) +// ) +// insertEdgesSync(bulkEdges: _*) // -// val bulkEdges = Seq( -// toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), -// toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), -// toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)), -// toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40)) -// ) -// insertEdgesSync(bulkEdges: _*) +// var result = getEdgesSync(querySingle(src, label, offset = 1, limit = 2)) // -// var result = getEdgesSync(querySingle(src, offset = 0, limit = 2)) -// var edges = (result \ "results").as[List[JsValue]] +// var edges = (result \ "results").as[List[JsValue]] // -// edges.size should be(2) -// (edges(0) \ "to").as[Long] should be(4) -// (edges(1) \ "to").as[Long] should be(3) +// edges.size should be(2) +// (edges(0) \ "to").as[Long] should be(4) +// (edges(1) \ "to").as[Long] should be(3) // -// result = getEdgesSync(querySingle(src, offset = 1, limit = 2)) +// result = getEdgesSync(querySingle(src, label, offset = 1, limit = 2)) // -// edges = (result \ "results").as[List[JsValue]] -// edges.size should be(2) -// (edges(0) \ "to").as[Long] should be(3) -// (edges(1) \ "to").as[Long] should be(2) +// edges = (result \ "results").as[List[JsValue]] +// edges.size should be(2) +// (edges(0) \ "to").as[Long] should be(3) +// (edges(1) \ "to").as[Long] should be(2) // -// result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1)) -// edges = (result \ "results").as[List[JsValue]] -// edges.size should be(1) -// } - - test("order by") { - def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj( - "srcVertices" -> Json.arr( - Json.obj( - "serviceName" -> testServiceName, - "columnName" -> testColumnName, - "id" -> id - ) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) - ) - ) - def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj( - "orderBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) +// result = getEdgesSync(querySingleWithTo(src, label, offset = 0, limit = -1, to = 1)) +// edges = (result \ "results").as[List[JsValue]] +// edges.size should be(1) +// } + + test(s"order by $ver", tag) { + val bulkEdges = Seq( + toEdge(1001, insert, e, 0, 1, label, Json.obj(weight -> 10, is_hidden -> true)), + toEdge(2002, insert, e, 0, 2, label, Json.obj(weight -> 20, is_hidden -> false)), + toEdge(3003, insert, e, 2, 0, label, Json.obj(weight -> 30)), + toEdge(4004, insert, e, 2, 1, label, Json.obj(weight -> 40)) ) - ) - - val bulkEdges = Seq( - toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), - toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), - toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)), - toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)) - ) - - insertEdgesSync(bulkEdges: _*) - - // get edges - val edges = getEdgesSync(queryScore(0, Map("weight" -> 1))) - val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) - val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) - - val edgesTo = edges \ "results" \\ "to" - val orderByTo = orderByScore \ "results" \\ "to" - val ascOrderByTo = ascOrderByScore \ "results" \\ "to" - - edgesTo should be(Seq(JsNumber(2), JsNumber(1))) - edgesTo should be(orderByTo) - ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2))) - edgesTo.reverse should be(ascOrderByTo) - } - - - test("query with sampling") { - def queryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - } - ] - }""") - - def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - }, - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - } - ] - }""") - - def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 50, - "sample": $sample - }, - { - "label": "$testLabelName2", - "direction": "out", - "offset": 0, - "limit": 50 - }] - } - ] - }""") - - val sampleSize = 2 - val ts = "1442985659166" - val testId = 22 - - val bulkEdges = Seq( - toEdge(ts, insert, e, testId, 122, testLabelName), - toEdge(ts, insert, e, testId, 222, testLabelName), - toEdge(ts, insert, e, testId, 322, testLabelName), - - toEdge(ts, insert, e, testId, 922, testLabelName2), - toEdge(ts, insert, e, testId, 222, testLabelName2), - toEdge(ts, insert, e, testId, 322, testLabelName2), - toEdge(ts, insert, e, 122, 1122, testLabelName), - toEdge(ts, insert, e, 122, 1222, testLabelName), - toEdge(ts, insert, e, 122, 1322, testLabelName), - toEdge(ts, insert, e, 222, 2122, testLabelName), - toEdge(ts, insert, e, 222, 2222, testLabelName), - toEdge(ts, insert, e, 222, 2322, testLabelName), - toEdge(ts, insert, e, 322, 3122, testLabelName), - toEdge(ts, insert, e, 322, 3222, testLabelName), - toEdge(ts, insert, e, 322, 3322, testLabelName) - ) + insertEdgesSync(bulkEdges: _*) - insertEdgesSync(bulkEdges: _*) + // get edges + val edges = getEdgesSync(queryScore(0, label, Map("weight" -> 1))) + val orderByScore = getEdgesSync(queryOrderBy(0, label, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) + val ascOrderByScore = getEdgesSync(queryOrderBy(0, label, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) - val result1 = getEdgesSync(queryWithSampling(testId, sampleSize)) - (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size)) + val edgesTo = edges \ "results" \\ "to" + val orderByTo = orderByScore \ "results" \\ "to" + val ascOrderByTo = ascOrderByScore \ "results" \\ "to" - val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize)) - (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) - - val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize)) - (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3 - } - test("test query with filterOut query") { - def queryWithFilterOut(id1: String, id2: String) = Json.parse( - s"""{ - | "limit": 10, - | "filterOut": { - | "srcVertices": [{ - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | }], - | "steps": [{ - | "step": [{ - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | }] - | }] - | }, - | "srcVertices": [{ - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | }], - | "steps": [{ - | "step": [{ - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | }] - | }] - |} - """.stripMargin - ) - - val testId1 = "-23" - val testId2 = "-25" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 111, testLabelName, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 333, testLabelName, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 555, testLabelName, Json.obj(weight -> 1)) - ) - logger.debug(s"${bulkEdges.mkString("\n")}") - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithFilterOut(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - (results(0) \ "to").toString should be("555") - } - - - /** note that this merge two different label result into one */ - test("weighted union") { - def queryWithWeightedUnion(id1: String, id2: String) = Json.parse( - s""" - |{ - | "limit": 10, - | "weights": [ - | 10, - | 1 - | ], - | "groupBy": ["weight"], - | "queries": [ - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | }, - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId1 = "1" - val testId2 = "2" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 444, testLabelName2, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 666, testLabelName2, Json.obj(weight -> 1)) - ) - - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithWeightedUnion(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - (results(0) \ "scoreSum").as[Float] should be(30.0) - (results(0) \ "agg").as[List[JsValue]].size should be(3) - (results(1) \ "scoreSum").as[Float] should be(3.0) - (results(1) \ "agg").as[List[JsValue]].size should be(3) - } - - test("weighted union with options") { - def queryWithWeightedUnionWithOptions(id1: String, id2: String) = Json.parse( - s""" - |{ - | "limit": 10, - | "weights": [ - | 10, - | 1 - | ], - | "groupBy": ["to"], - | "select": ["to", "weight"], - | "filterOut": { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | } - | ] - | }, - | "queries": [ - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | }, - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId1 = "-192848" - val testId2 = "-193849" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 111, testLabelName2, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 333, testLabelName2, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)) - ) - - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithWeightedUnionWithOptions(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - - } - - test("scoreThreshold") { - def queryWithScoreThreshold(id: String, scoreThreshold: Int) = Json.parse( - s"""{ - | "limit": 10, - | "scoreThreshold": $scoreThreshold, - | "groupBy": ["to"], - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | }, - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId = "-23903" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 102, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 104, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 102, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 102, 104, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 103, 105, testLabelName, Json.obj(weight -> 10)) - ) - // expected: 104 -> 2, 103 -> 2, 102 -> 1,, 105 -> 1 - insertEdgesSync(bulkEdges: _*) - - var rs = getEdgesSync(queryWithScoreThreshold(testId, 2)) - logger.debug(Json.prettyPrint(rs)) - var results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - - rs = getEdgesSync(queryWithScoreThreshold(testId, 1)) - logger.debug(Json.prettyPrint(rs)) - - results = (rs \ "results").as[List[JsValue]] - results.size should be(4) - } - - test("scorePropagateOp test") { - def queryWithPropertyOp(id: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "index": "idx_1", - | "scoring": { - | "weight":1, - | "time": 0 - | }, - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - def querySingleVertexWithOp(id: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "countSum", - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - def queryMultiVerticesWithOp(id: String, id2: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "ids": [$id, $id2] - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "countSum", - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - val testId = "-30000" - val testId2 = "-4000" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 102, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 103, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 104, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 105, testLabelName2, Json.obj(weight -> 10)), + edgesTo should be(Seq(JsNumber(2), JsNumber(1))) + edgesTo should be(orderByTo) + ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2))) + edgesTo.reverse should be(ascOrderByTo) + } - toEdge(1, insert, e, testId2, 101, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 102, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 103, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 102, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId2, 105, testLabelName2, Json.obj(weight -> 10)) - ) - insertEdgesSync(bulkEdges: _*) + test(s"query with sampling $ver", tag) { + + val sampleSize = 2 + val ts = "1442985659166" + val testId = 22 + + val bulkEdges = Seq( + toEdge(ts, insert, e, testId, 122, label), + toEdge(ts, insert, e, testId, 222, label), + toEdge(ts, insert, e, testId, 322, label), + + toEdge(ts, insert, e, testId, 922, label2), + toEdge(ts, insert, e, testId, 222, label2), + toEdge(ts, insert, e, testId, 322, label2), + + toEdge(ts, insert, e, 122, 1122, label), + toEdge(ts, insert, e, 122, 1222, label), + toEdge(ts, insert, e, 122, 1322, label), + toEdge(ts, insert, e, 222, 2122, label), + toEdge(ts, insert, e, 222, 2222, label), + toEdge(ts, insert, e, 222, 2322, label), + toEdge(ts, insert, e, 322, 3122, label), + toEdge(ts, insert, e, 322, 3222, label), + toEdge(ts, insert, e, 322, 3322, label) + ) - val firstStepEdgeCount = 3l - val secondStepEdgeCount = 4l + insertEdgesSync(bulkEdges: _*) + val result0 = getEdgesSync(querySingle(testId, label)) + logger.debug(s"2-step sampling res 0: ${Json.prettyPrint(result0)}") + (result0 \ "results").as[List[JsValue]].size should be(3) - var shrinkageVal = 10l - var rs = getEdgesSync(querySingleVertexWithOp(testId, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - var results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - var scoreSum = secondStepEdgeCount.toDouble / (firstStepEdgeCount.toDouble + shrinkageVal) - (results(0) \ "scoreSum").as[Double] should be(scoreSum) + val result1 = getEdgesSync(queryWithSampling(testId, label, sampleSize)) + logger.debug(s"2-step sampling res 1: ${Json.prettyPrint(result1)}") + (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size)) - rs = getEdgesSync(queryMultiVerticesWithOp(testId, testId2, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - scoreSum = secondStepEdgeCount.toDouble / (firstStepEdgeCount.toDouble + shrinkageVal) - (results(0) \ "scoreSum").as[Double] should be(scoreSum) - scoreSum = 2.toDouble / (3.toDouble + shrinkageVal) - (results(1) \ "scoreSum").as[Double] should be(scoreSum) + val result2 = getEdgesSync(twoStepQueryWithSampling(testId, label, sampleSize)) + logger.debug(s"2-step sampling res 2: ${Json.prettyPrint(result2)}") + (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) - // check for divide zero case - shrinkageVal = 30l - rs = getEdgesSync(queryWithPropertyOp(testId, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - (results(0) \ "scoreSum").as[Double] should be(0) + val result3 = getEdgesSync(twoQueryWithSampling(testId, label, label2, sampleSize)) + logger.debug(s"2-step sampling res 3: ${Json.prettyPrint(result3)}") + (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3 + } - // "plus" operation - rs = getEdgesSync(querySingleVertexWithOp(testId, "plus", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - scoreSum = (firstStepEdgeCount + 1) * secondStepEdgeCount - (results(0) \ "scoreSum").as[Long] should be(scoreSum) - // "multiply" operation - rs = getEdgesSync(querySingleVertexWithOp(testId, "multiply", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - scoreSum = (firstStepEdgeCount * 1) * secondStepEdgeCount - (results(0) \ "scoreSum").as[Long] should be(scoreSum) + // "checkEdges" in { + // running(FakeApplication()) { + // val json = Json.parse( s""" + // [{"from": 0, "to": 1, "label": "$testLabelName"}, + // {"from": 0, "to": 2, "label": "$testLabelName"}] + // """) + // + // def checkEdges(queryJson: JsValue): JsValue = { + // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get + // contentAsJson(ret) + // } + // + // val res = checkEdges(json) + // val typeRes = res.isInstanceOf[JsArray] + // typeRes must equalTo(true) + // + // val fst = res.as[Seq[JsValue]].head \ "to" + // fst.as[Int] must equalTo(1) + // + // val snd = res.as[Seq[JsValue]].last \ "to" + // snd.as[Int] must equalTo(2) + // } + // } } - def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "direction": "out", - "offset": $offset, - "limit": $limit - } - ]] - } - """) - - def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj( - "limit" -> limit, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName - ) - ) - ) - ) - ) - - // called by each test, each override def beforeEach = initTestData() @@ -1097,25 +246,35 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { override def initTestData(): Unit = { super.initTestData() - insertEdgesSync( - toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)), - toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)), - toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)), - toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)), - toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)), - toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, -1, 1000, testLabelName), - toEdge(1, insert, e, -1, 2000, testLabelName), - toEdge(1, insert, e, -1, 3000, testLabelName), - toEdge(1, insert, e, 1000, 10000, testLabelName), - toEdge(1, insert, e, 1000, 11000, testLabelName), - toEdge(1, insert, e, 2000, 11000, testLabelName), - toEdge(1, insert, e, 2000, 12000, testLabelName), - toEdge(1, insert, e, 3000, 12000, testLabelName), - toEdge(1, insert, e, 3000, 13000, testLabelName), - toEdge(1, insert, e, 10000, 100000, testLabelName), - toEdge(2, insert, e, 11000, 200000, testLabelName), - toEdge(3, insert, e, 12000, 300000, testLabelName) - ) + val vers = config.getString("s2graph.storage.backend") match { + case "hbase" => versions + case "redis" => Seq(4) + case _ => throw new RuntimeException("not supported storage.") + } + + vers map { n => + val ver = s"v$n" + val label = getLabelName(ver) + insertEdgesSync( + toEdge(1000, insert, e, 0, 1, label, Json.obj(weight -> 40, is_hidden -> true)), + toEdge(2000, insert, e, 0, 2, label, Json.obj(weight -> 30, is_hidden -> false)), + toEdge(3000, insert, e, 2, 0, label, Json.obj(weight -> 20)), + toEdge(4000, insert, e, 2, 1, label, Json.obj(weight -> 10)), + toEdge(3000, insert, e, 10, 20, label, Json.obj(weight -> 20)), + toEdge(4000, insert, e, 20, 20, label, Json.obj(weight -> 10)), + toEdge(1, insert, e, -1, 1000, label), + toEdge(1, insert, e, -1, 2000, label), + toEdge(1, insert, e, -1, 3000, label), + toEdge(1, insert, e, 1000, 10000, label), + toEdge(1, insert, e, 1000, 11000, label), + toEdge(1, insert, e, 2000, 11000, label), + toEdge(1, insert, e, 2000, 12000, label), + toEdge(1, insert, e, 3000, 12000, label), + toEdge(1, insert, e, 3000, 13000, label), + toEdge(1, insert, e, 10000, 100000, label), + toEdge(2, insert, e, 11000, 200000, label), + toEdge(3, insert, e, 12000, 300000, label) + ) + } } } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala index 99b56f7b..f6e83d91 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -32,198 +32,205 @@ class StrongLabelDeleteTest extends IntegrateCommon { import StrongDeleteUtil._ import TestUtil._ - test("Strong consistency select") { - insertEdgesSync(bulkEdges(): _*) + override val versions = 2 to 4 // StrongLabelDeleteTest is supported by v2 and above. + versions map { n => - var result = getEdgesSync(query(0)) - (result \ "results").as[List[JsValue]].size should be(2) - result = getEdgesSync(query(10)) - (result \ "results").as[List[JsValue]].size should be(2) - } - - test("Strong consistency deleteAll") { - val deletedAt = 100 - var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) + val ver = s"v$n" + val label = getLabelName(ver) + val tag = getTag(ver) - println(result) - (result \ "results").as[List[JsValue]].size should be(3) + test(s"Strong consistency select $ver", tag) { + insertEdgesSync(bulkEdges(label = label): _*) - val deleteParam = Json.arr( - Json.obj("label" -> testLabelName2, - "direction" -> "in", - "ids" -> Json.arr("20"), - "timestamp" -> deletedAt)) + var result = getEdgesSync(query(0, label)) + (result \ "results").as[List[JsValue]].size should be(2) + result = getEdgesSync(query(10, label)) + (result \ "results").as[List[JsValue]].size should be(2) + } - deleteAllSync(deleteParam) + test(s"Strong consistency deleteAll $ver", tag) { + val deletedAt = 100 + var result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) - result = getEdgesSync(query(11, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + // println(result) + (result \ "results").as[List[JsValue]].size should be(3) - result = getEdgesSync(query(12, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + val deleteParam = Json.arr( + Json.obj("label" -> getLabelName(ver), + "direction" -> "in", + "ids" -> Json.arr("20"), + "timestamp" -> deletedAt)) - result = getEdgesSync(query(10, direction = "out")) - println(result) - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size should be(1) - (result \\ "to").size should be(1) - (result \\ "to").head.as[String] should be("21") + deleteAllSync(deleteParam) - result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(query(11, label, direction = "out")) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) - insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*) + result = getEdgesSync(query(12, label, direction = "out")) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) + result = getEdgesSync(query(10, label, direction = "out")) + // println(result) + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size should be(1) + (result \\ "to").size should be(1) + (result \\ "to").head.as[Long] should be(21l) - (result \ "results").as[List[JsValue]].size should be(3) - } + result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) + insertEdgesSync(bulkEdges(startTs = deletedAt + 1, label): _*) - test("update delete") { - val ret = for { - i <- 0 until testNum - } yield { - val src = (i + 1) * 10000 -// val src = System.currentTimeMillis() + result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) + // println(result) - val (ret, last) = testInner(i, src) - ret should be(true) - ret + (result \ "results").as[List[JsValue]].size should be(3) } - ret.forall(identity) - } - test("update delete 2") { - val src = System.currentTimeMillis() - var ts = 0L + test(s"update delete $ver", tag) { + val ret = for { + i <- 0 until testNum + } yield { + val src = System.currentTimeMillis() - val ret = for { - i <- 0 until testNum - } yield { - val (ret, lastTs) = testInner(ts, src) - val deletedAt = lastTs + 1 - val deletedAt2 = lastTs + 2 - ts = deletedAt2 + 1 // nex start ts + val (ret, last) = testInner(i, src, label) + // val (ret, last) = testInnerFail(i, src) + ret should be(true) + ret + } - ret should be(true) + ret.forall(identity) + } - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) + test(s"update delete 2 $ver", tag) { + val src = System.currentTimeMillis() + var ts = 0L - val deleteRet = deleteAllSync(deleteAllRequest) - val deleteRet2 = deleteAllSync(deleteAllRequest2) + val ret = for { + i <- 0 until testNum + } yield { + val (ret, lastTs) = testInner(ts, src, label) + val deletedAt = lastTs + 1 + val deletedAt2 = lastTs + 2 + ts = deletedAt2 + 1 // nex start ts - val result = getEdgesSync(query(id = src)) - println(result) + ret should be(true) - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty should be(true) + val deleteAllRequest = Json.arr(Json.obj("label" -> label, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + val deleteAllRequest2 = Json.arr(Json.obj("label" -> label, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) - val degreeAfterDeleteAll = getDegree(result) + val deleteRet = deleteAllSync(deleteAllRequest) + val deleteRet2 = deleteAllSync(deleteAllRequest2) - degreeAfterDeleteAll should be(0) - degreeAfterDeleteAll === (0) - } + val result = getEdgesSync(query(id = src, label)) + // println(result) - ret.forall(identity) - } + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty should be(true) - /** This test stress out test on degree - * when contention is low but number of adjacent edges are large - * Large set of contention test - */ - test("large degrees") { - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - // val allRequests = insertRequests ++ deleteRequests - val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => - insertEdgesAsync(bulkRequests: _*) + val degreeAfterDeleteAll = getDegree(result) + + degreeAfterDeleteAll should be(0) + degreeAfterDeleteAll === (0) + } + + ret.forall(identity) } - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + /** This test stress out test on degree + * when contention is low but number of adjacent edges are large + * Large set of contention test + */ + test(s"large degrees $ver", tag) { + val labelName = getLabelName(ver) + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + // val allRequests = insertRequests ++ deleteRequests + val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => + insertEdgesAsync(bulkRequests: _*) + } - val expectedDegree = insertRequests.size - deleteRequests.size - val queryJson = query(id = src) - val result = getEdgesSync(queryJson) - val resultSize = (result \ "size").as[Long] - val resultDegree = getDegree(result) + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - // println(result) + val expectedDegree = insertRequests.size - deleteRequests.size + val queryJson = query(id = src, label) + val result = getEdgesSync(queryJson) + val resultSize = (result \ "size").as[Long] + val resultDegree = getDegree(result) - val ret = resultSize == expectedDegree && resultDegree == resultSize - println(s"[MaxSize]: $maxSize") - println(s"[DeleteSize]: $deleteSize") - println(s"[ResultDegree]: $resultDegree") - println(s"[ExpectedDegree]: $expectedDegree") - println(s"[ResultSize]: $resultSize") - ret should be(true) - } + // println(result) - test("deleteAll") { - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => - insertEdgesAsync(bulkRequests: _*) + val ret = resultSize == expectedDegree && resultDegree == resultSize + // println(s"[MaxSize]: $maxSize") + // println(s"[DeleteSize]: $deleteSize") + // println(s"[ResultDegree]: $resultDegree") + // println(s"[ExpectedDegree]: $expectedDegree") + // println(s"[ResultSize]: $resultSize") + ret should be(true) } - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + test(s"deleteAll $ver", tag) { + val labelName = getLabelName(ver) + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => + insertEdgesAsync(bulkRequests: _*) + } - val deletedAt = System.currentTimeMillis() - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - deleteAllSync(deleteAllRequest) + // val deletedAt = System.currentTimeMillis() + val deletedAt = src + maxSize + 1000 + 1000 + val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - val result = getEdgesSync(query(id = src)) - println(result) - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty should be(true) + deleteAllSync(deleteAllRequest) - val degreeAfterDeleteAll = getDegree(result) - degreeAfterDeleteAll should be(0) + val result = getEdgesSync(query(id = src, label)) + // println(s"!!!!!RESULT: ${Json.prettyPrint(result)}") + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty should be(true) + + val degreeAfterDeleteAll = getDegree(result) + degreeAfterDeleteAll should be(0) + } } object StrongDeleteUtil { - val labelName = testLabelName2 -// val labelName = testLabelName val maxTgtId = 10 val batchSize = 10 - val testNum = 100 + val testNum = 3 val numOfBatch = 10 - def testInner(startTs: Long, src: Long) = { + def testInner(startTs: Long, src: Long, label: String) = { val lastOps = Array.fill(maxTgtId)("none") var currentTs = startTs @@ -237,7 +244,7 @@ class StrongLabelDeleteTest extends IntegrateCommon { val op = if (Random.nextDouble() < 0.5) "delete" else "update" lastOps(tgt) = op - Seq(currentTs, op, "e", src, tgt, labelName, "{}").mkString("\t") + Seq(currentTs, op, "e", src, tgt, label, "{}").mkString("\t") } allRequests.foreach(println(_)) @@ -249,7 +256,7 @@ class StrongLabelDeleteTest extends IntegrateCommon { Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) val expectedDegree = lastOps.count(op => op != "delete" && op != "none") - val queryJson = query(id = src) + val queryJson = query(id = src, label = label) val result = getEdgesSync(queryJson) val resultSize = (result \ "size").as[Long] val resultDegree = getDegree(result) @@ -263,19 +270,19 @@ class StrongLabelDeleteTest extends IntegrateCommon { (ret, currentTs) } - def bulkEdges(startTs: Int = 0) = Seq( - toEdge(startTs + 1, "insert", "e", "0", "1", labelName, s"""{"time": 10}"""), - toEdge(startTs + 2, "insert", "e", "0", "1", labelName, s"""{"time": 11}"""), - toEdge(startTs + 3, "insert", "e", "0", "1", labelName, s"""{"time": 12}"""), - toEdge(startTs + 4, "insert", "e", "0", "2", labelName, s"""{"time": 10}"""), - toEdge(startTs + 5, "insert", "e", "10", "20", labelName, s"""{"time": 10}"""), - toEdge(startTs + 6, "insert", "e", "10", "21", labelName, s"""{"time": 11}"""), - toEdge(startTs + 7, "insert", "e", "11", "20", labelName, s"""{"time": 12}"""), - toEdge(startTs + 8, "insert", "e", "12", "20", labelName, s"""{"time": 13}""") + def bulkEdges(startTs: Int = 0, label: String) = Seq( + toEdge(startTs + 1, "insert", "e", "0", "1", label, s"""{"time": 10}"""), + toEdge(startTs + 2, "insert", "e", "0", "1", label, s"""{"time": 11}"""), + toEdge(startTs + 3, "insert", "e", "0", "1", label, s"""{"time": 12}"""), + toEdge(startTs + 4, "insert", "e", "0", "2", label, s"""{"time": 10}"""), + toEdge(startTs + 5, "insert", "e", "10", "20", label, s"""{"time": 10}"""), + toEdge(startTs + 6, "insert", "e", "10", "21", label, s"""{"time": 11}"""), + toEdge(startTs + 7, "insert", "e", "11", "20", label, s"""{"time": 12}"""), + toEdge(startTs + 8, "insert", "e", "12", "20", label, s"""{"time": 13}""") ) - def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName, - _labelName: String = labelName, direction: String = "out") = Json.parse( + def query(id: Long, label: String, serviceName: String = testServiceName, columnName: String = testColumnName, + direction: String = "out") = Json.parse( s""" { "srcVertices": [ { "serviceName": "$serviceName", @@ -284,7 +291,7 @@ class StrongLabelDeleteTest extends IntegrateCommon { }], "steps": [ [ { - "label": "${_labelName}", + "label": "${label}", "direction": "${direction}", "offset": 0, "limit": -1, diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala similarity index 64% rename from s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala rename to s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala index a1bff685..e1cedec7 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,25 +19,23 @@ package org.apache.s2graph.core.Integrate -import org.apache.s2graph.core.PostProcess +import org.apache.s2graph.core.{CommonTest, PostProcess} import play.api.libs.json.{JsValue, Json} import scala.concurrent.Await -import scala.util.Random -class VertexTestHelper extends IntegrateCommon { +class VertexTest extends IntegrateCommon { import TestUtil._ - import VertexTestHelper._ - test("vertex") { + test("vertex", CommonTest) { val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0) val (serviceName, columnName) = (testServiceName, testColumnName) val data = vertexInsertsPayload(serviceName, columnName, ids) val payload = Json.parse(Json.toJson(data).toString) - println(payload) + println(s">>> insert vertices: ${payload}") val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName)) Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime) @@ -48,6 +46,7 @@ class VertexTestHelper extends IntegrateCommon { val ret = Await.result(res, HttpRequestWaitingTime) val fetched = ret.as[Seq[JsValue]] + println(s">>> fetched vertices: ${fetched}") for { (d, f) <- data.zip(fetched) } yield { @@ -55,36 +54,6 @@ class VertexTestHelper extends IntegrateCommon { ((d \ "props") \ "age") should be((f \ "props") \ "age") } } - - object VertexTestHelper { - def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { - Json.parse( - s""" - |[ - |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")} - ]} - |] - """.stripMargin) - } - - def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { - ids.map { id => - Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) - } - } - - val vertexPropsKeys = List( - ("age", "int") - ) - - def randomProps() = { - (for { - (propKey, propType) <- vertexPropsKeys - } yield { - propKey -> Random.nextInt(100) - }).toMap - } - } } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala index c0ab3234..0b030343 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -32,83 +32,92 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { import TestUtil._ import WeakLabelDeleteHelper._ - test("test weak consistency select") { - var result = getEdgesSync(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size should be(4) - result = getEdgesSync(query(10)) - println(result) - (result \ "results").as[List[JsValue]].size should be(2) - } - - test("test weak consistency delete") { - var result = getEdgesSync(query(0)) - println(result) - - /** expect 4 edges */ - (result \ "results").as[List[JsValue]].size should be(4) - val edges = (result \ "results").as[List[JsObject]] - val edgesToStore = parser.toEdges(Json.toJson(edges), "delete") - val rets = graph.mutateEdges(edgesToStore, withWait = true) - Await.result(rets, Duration(20, TimeUnit.MINUTES)) - - /** expect noting */ - result = getEdgesSync(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) - - /** insert should be ignored */ - /** - * I am wondering if this is right test case - * This makes sense because hbase think cell is deleted when there are - * insert/delete with same timestamp(version) on same cell. - * This can be different on different storage system so I think - * this test should be removed. - */ -// val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert") -// val rets2 = graph.mutateEdges(edgesToStore2, withWait = true) -// Await.result(rets2, Duration(20, TimeUnit.MINUTES)) + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + val label = getLabelName(ver, "weak") + + test(s"test weak consistency select $ver", tag) { + var result = getEdgesSync(queryWeak(0, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(4) + result = getEdgesSync(queryWeak(10, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(2) + } + + test(s"test weak consistency delete $ver", tag) { + var result = getEdgesSync(queryWeak(0, label)) + println(result) + + /** expect 4 edges */ + (result \ "results").as[List[JsValue]].size should be(4) + + val edges = (result \ "results").as[List[JsObject]] + val edgesToStore = parser.toEdges(Json.toJson(edges), "delete") + val rets = graph.mutateEdges(edgesToStore, withWait = true) + Await.result(rets, Duration(20, TimeUnit.MINUTES)) + + /** expect noting */ + result = getEdgesSync(queryWeak(0, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(0) + } + + /** insert should be ignored */ + /** + * I am wondering if this is right test case + * This makes sense because hbase think cell is deleted when there are + * insert/delete with same timestamp(version) on same cell. + * This can be different on different storage system so I think + * this test should be removed. => Indeed, Redis does not support this feature. + */ +// val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert") +// val rets2 = graph.mutateEdges(edgesToStore2, withWait = true) +// Await.result(rets2, Duration(20, TimeUnit.MINUTES)) // -// result = getEdgesSync(query(0)) -// (result \ "results").as[List[JsValue]].size should be(0) - } +// result = getEdgesSync(queryWeak(0, label)) +// (result \ "results").as[List[JsValue]].size should be(0) - test("test weak consistency deleteAll") { - val deletedAt = 100 - var result = getEdgesSync(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(3) + test(s"test weak consistency deleteAll $ver", tag) { + val deletedAt = 100 + var result = getEdgesSync(queryWeak(20, label, "in")) + println(result) + (result \ "results").as[List[JsValue]].size should be(3) - val json = Json.arr(Json.obj("label" -> testLabelNameWeak, - "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) - println(json) - deleteAllSync(json) + val json = Json.arr(Json.obj("label" -> label, + "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) + println(json) + deleteAllSync(json) - result = getEdgesSync(query(11, "out")) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(11, label, "out")) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(12, "out")) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(12, label, "out")) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(10, "out")) + result = getEdgesSync(queryWeak(10, label, "out")) - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size should be(1) - (result \\ "to").size should be(1) - (result \\ "to").head.as[String] should be("21") + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size should be(1) + (result \\ "to").size should be(1) + (result \\ "to").head.as[Long] should be(21L) - result = getEdgesSync(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(20, label, "in")) + println(result) + (result \ "results").as[List[JsValue]].size should be(0) - insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*) + insertEdgesSync(bulkEdges(startTs = deletedAt + 1, label): _*) - result = getEdgesSync(query(20, "in", testTgtColumnName)) - (result \ "results").as[List[JsValue]].size should be(3) + result = getEdgesSync(queryWeak(20, label, "in", testColumnName)) + (result \ "results").as[List[JsValue]].size should be(3) + } } + + // called by each test, each override def beforeEach = initTestData() @@ -116,23 +125,26 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { override def initTestData(): Unit = { super.initTestData() - insertEdgesSync(bulkEdges(): _*) + versions map { v => + val ver = s"v$v" + insertEdgesSync(bulkEdges(label = getLabelName(ver, "weak")): _*) + } } object WeakLabelDeleteHelper { - def bulkEdges(startTs: Int = 0) = Seq( - toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""), - toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""), - toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""), - toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""), - toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""") + def bulkEdges(startTs: Int = 0, label: String) = Seq( + toEdge(startTs + 1, "insert", "e", "0", "1", label, s"""{"time": 10}"""), + toEdge(startTs + 2, "insert", "e", "0", "1", label, s"""{"time": 11}"""), + toEdge(startTs + 3, "insert", "e", "0", "1", label, s"""{"time": 12}"""), + toEdge(startTs + 4, "insert", "e", "0", "2", label, s"""{"time": 10}"""), + toEdge(startTs + 5, "insert", "e", "10", "20", label, s"""{"time": 10}"""), + toEdge(startTs + 6, "insert", "e", "10", "21", label, s"""{"time": 11}"""), + toEdge(startTs + 7, "insert", "e", "11", "20", label, s"""{"time": 12}"""), + toEdge(startTs + 8, "insert", "e", "12", "20", label, s"""{"time": 13}""") ) - def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse( + def queryWeak(id: Int, label: String, direction: String = "out", columnName: String = testColumnName) = Json.parse( s""" { "srcVertices": [ { "serviceName": "$testServiceName", @@ -141,7 +153,7 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { }], "steps": [ [ { - "label": "${testLabelNameWeak}", + "label": "${label}", "direction": "${direction}", "offset": 0, "limit": 10, @@ -153,3 +165,4 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { } + diff --git a/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala index 03151882..47183abc 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,38 +22,53 @@ package org.apache.s2graph.core import org.apache.s2graph.core.types.{InnerVal, InnerValLike} import org.scalatest.{FunSuite, Matchers} -class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser { +class JsonParserTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels with JSONParser { import InnerVal._ - import types.HBaseType._ + import types.GraphType._ - val innerValsPerVersion = for { - version <- List(VERSION2, VERSION1) - } yield { - val innerVals = List( - (InnerVal.withStr("ABC123", version), STRING), - (InnerVal.withNumber(23, version), BYTE), - (InnerVal.withNumber(23, version), INT), - (InnerVal.withNumber(Int.MaxValue, version), INT), - (InnerVal.withNumber(Int.MinValue, version), INT), - (InnerVal.withNumber(Long.MaxValue, version), LONG), - (InnerVal.withNumber(Long.MinValue, version), LONG), - (InnerVal.withBoolean(true, version), BOOLEAN) + versions map { n => + + val ver = s"v$n" + val tag = getTag(ver) + + test(s"aa $ver", tag) { + val innerVal = InnerVal.withStr("abc", ver) + val tmp = innerValToJsValue(innerVal, "string") + println(tmp) + } + + test(s"JsValue <-> InnerVal with dataType $ver", tag) { + val (innerValWithDataTypes, version) = innerValsPerVersion(ver) + testInnerValToJsValue(innerValWithDataTypes, version) + } + } + + def innerValsPerVersion(version: String) = { + val innerVals = List( + (InnerVal.withStr("ABC123", version), STRING), + (InnerVal.withNumber(23, version), BYTE), + (InnerVal.withNumber(23, version), INT), + (InnerVal.withNumber(Int.MaxValue, version), INT), + (InnerVal.withNumber(Int.MinValue, version), INT), + (InnerVal.withNumber(Long.MaxValue, version), LONG), + (InnerVal.withNumber(Long.MinValue, version), LONG), + (InnerVal.withBoolean(true, version), BOOLEAN) + ) + val doubleVals = if (version != VERSION1) { + List( + (InnerVal.withDouble(Double.MaxValue, version), DOUBLE), + (InnerVal.withDouble(Double.MinValue, version), DOUBLE), + (InnerVal.withDouble(0.1, version), DOUBLE), + (InnerVal.withFloat(Float.MinValue, version), FLOAT), + (InnerVal.withFloat(Float.MaxValue, version), FLOAT), + (InnerVal.withFloat(0.9f, version), FLOAT) ) - val doubleVals = if (version == VERSION2) { - List( - (InnerVal.withDouble(Double.MaxValue, version), DOUBLE), - (InnerVal.withDouble(Double.MinValue, version), DOUBLE), - (InnerVal.withDouble(0.1, version), DOUBLE), - (InnerVal.withFloat(Float.MinValue, version), FLOAT), - (InnerVal.withFloat(Float.MaxValue, version), FLOAT), - (InnerVal.withFloat(0.9f, version), FLOAT) - ) - } else { - List.empty[(InnerValLike, String)] - } - (innerVals ++ doubleVals, version) + } else { + List.empty[(InnerValLike, String)] } + (innerVals ++ doubleVals, version) + } def testInnerValToJsValue(innerValWithDataTypes: Seq[(InnerValLike, String)], version: String) = { @@ -70,16 +85,5 @@ class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONPar } } - test("aa") { - val innerVal = InnerVal.withStr("abc", VERSION2) - val tmp = innerValToJsValue(innerVal, "string") - println(tmp) - } - test("JsValue <-> InnerVal with dataType") { - for { - (innerValWithDataTypes, version) <- innerValsPerVersion - } { - testInnerValToJsValue(innerValWithDataTypes, version) - } - } + } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala index c6de6bba..610b0384 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,7 +24,7 @@ import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.JsString class OrderingUtilTest extends FunSuite with Matchers { - test("test SeqMultiOrdering") { + test("test SeqMultiOrdering", CommonTest) { val jsLs: Seq[Seq[Any]] = Seq( Seq(0, "a"), Seq(0, "b"), @@ -48,7 +48,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 1 TupleMultiOrdering") { + test("test tuple 1 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, None, None, None), (0, None, None, None), @@ -71,7 +71,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 2 TupleMultiOrdering") { + test("test tuple 2 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", None, None), (0, "b", None, None), @@ -95,7 +95,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 3 TupleMultiOrdering") { + test("test tuple 3 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", 0l, None), (0, "a", 1l, None), @@ -120,7 +120,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 4 TupleMultiOrdering") { + test("test tuple 4 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", 0l, JsString("a")), (0, "a", 0l, JsString("b")), @@ -147,3 +147,4 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } } + diff --git a/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala index 06af38cc..da3af922 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,20 +24,6 @@ import org.apache.s2graph.core.types.LabelWithDirection import org.scalatest.{FunSuite, Matchers} class QueryParamTest extends FunSuite with Matchers with TestCommon { -// val version = HBaseType.VERSION2 -// val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString) -// test("EdgeTransformer toInnerValOpt") { -// -// /** only labelNameV2 has string type output */ -// val jsVal = Json.arr(Json.arr("_to"), Json.arr("phone_number.$", "phone_number"), Json.arr("age.$", "age")) -// val transformer = EdgeTransformer(queryParamV2, jsVal) -// val convertedLs = transformer.transform(testEdge, None) -// -// convertedLs(0).tgtVertex.innerId.toString == "10" shouldBe true -// convertedLs(1).tgtVertex.innerId.toString == "phone_number.xxxx" shouldBe true -// convertedLs(2).tgtVertex.innerId.toString == "age.20" shouldBe true -// true -// } val dummyRequests = { for { @@ -47,7 +33,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { } } - test("QueryParam toCacheKey bytes") { + test("QueryParam toCacheKey bytes", CommonTest) { val startedAt = System.nanoTime() val queryParam = QueryParam(LabelWithDirection(1, 0)) @@ -66,15 +52,15 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { dummyRequests.zip(dummyRequests).foreach { case (x, y) => val xHash = queryParam.toCacheKey(x) val yHash = queryParam.toCacheKey(y) -// println(xHash, yHash) + // println(xHash, yHash) xHash should be(yHash) } val duration = System.nanoTime() - startedAt - println(s">> bytes: $duration") + // println(s">> bytes: $duration") } - test("QueryParam toCacheKey with variable params") { + test("QueryParam toCacheKey with variable params", CommonTest) { val startedAt = System.nanoTime() val queryParam = QueryParam(LabelWithDirection(1, 0)) @@ -86,7 +72,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { queryParam.limit(1, 10) var yHash = queryParam.toCacheKey(y) queryParam.toCacheKey(x) shouldBe yHash -// println(xHash, yHash) + // println(xHash, yHash) xHash should not be yHash queryParam.limit(0, 10) @@ -99,7 +85,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { val duration = System.nanoTime() - startedAt - println(s">> diff: $duration") + // println(s">> diff: $duration") } } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala index 0f8f8338..4a588151 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,12 +19,26 @@ package org.apache.s2graph.core +import com.typesafe.config.{ConfigFactory, Config} import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta} -import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLikeWithTs, LabelWithDirection} +import org.apache.s2graph.core.rest.RequestParser +import org.apache.s2graph.core.types.{GraphType, InnerVal, InnerValLikeWithTs, LabelWithDirection} trait TestCommon { + + var graph: Graph = _ + var config: Config = ConfigFactory.load() + var management: Management = _ + var parser: RequestParser = _ + + val versions = config.getString("s2graph.storage.backend") match { + case "hbase" => Seq(1,2,3,4) + case "redis" => Seq(4) + case _ => throw new RuntimeException("unsupported storage") + } + val ts = System.currentTimeMillis() val testServiceId = 1 val testColumnId = 1 @@ -43,8 +57,19 @@ trait TestCommon { def lessThanEqual(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) <= 0 + def getTag(ver: String) = { + ver match { + case "v1" => V1Test + case "v2" => V2Test + case "v3" => V3Test + case "v4" => V4Test + case _ => throw new GraphExceptions.UnsupportedVersionException(s"$ver does no support CRUD!") + } + } + + /** */ - import HBaseType.{VERSION1, VERSION2} + import GraphType.{VERSION1, VERSION2} private val tsValSmall = InnerVal.withLong(ts, VERSION1) private val tsValLarge = InnerVal.withLong(ts + 1, VERSION1) private val boolValSmall = InnerVal.withBoolean(false, VERSION1) diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala index c652e807..22286076 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,7 +19,6 @@ package org.apache.s2graph.core -import com.typesafe.config.{Config, ConfigFactory} import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} import org.apache.s2graph.core.mysqls.{Label, LabelIndex, Service, ServiceColumn} import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection} @@ -27,27 +26,22 @@ import scalikejdbc.AutoSession import scala.concurrent.ExecutionContext -trait TestCommonWithModels { +trait TestCommonWithModels extends TestCommon { import InnerVal._ - import types.HBaseType._ - var graph: Graph = _ - var config: Config = _ - var management: Management = _ - - def initTests() = { - config = ConfigFactory.load() + def initTests(ver: String) = { +// config = ConfigFactory.load() graph = new Graph(config)(ExecutionContext.Implicits.global) management = new Management(graph) implicit val session = AutoSession - deleteTestLabel() - deleteTestService() + deleteTestLabel(ver) + deleteTestService(ver) - createTestService() - createTestLabel() + createTestService(ver) + createTestLabel(ver) } def zkQuorum = config.getString("hbase.zookeeper.quorum") @@ -56,42 +50,19 @@ trait TestCommonWithModels { implicit val session = AutoSession - val serviceName = "_test_service" - val serviceNameV2 = "_test_service_v2" - val serviceNameV3 = "_test_service_v3" - val serviceNameV4 = "_test_service_v4" + def serviceName(ver: String) = s"_test_service_$ver" + def labelName(ver: String) = s"_test_label_$ver" + def undirectedLabelName(ver: String) = s"_test_label_undirected_$ver" - val columnName = "user_id" - val columnNameV2 = "user_id_v2" - val columnNameV3 = "user_id_v3" - val columnNameV4 = "user_id_v4" + def columnName(ver: String) = s"user_id_$ver" + def tgtColumnName(ver: String) = s"itme_id_$ver" val columnType = "long" - val columnTypeV2 = "long" - val columnTypeV3 = "long" - val columnTypeV4 = "long" - - val tgtColumnName = "itme_id" - val tgtColumnNameV2 = "item_id_v2" - val tgtColumnNameV3 = "item_id_v3" - val tgtColumnNameV4 = "item_id_v4" - val tgtColumnType = "string" - val tgtColumnTypeV2 = "string" - val tgtColumnTypeV3 = "string" - val tgtColumnTypeV4 = "string" val hTableName = "_test_cases" val preSplitSize = 0 - val labelName = "_test_label" - val labelNameV2 = "_test_label_v2" - val labelNameV3 = "_test_label_v3" - val labelNameV4 = "_test_label_v4" - - val undirectedLabelName = "_test_label_undirected" - val undirectedLabelNameV2 = "_test_label_undirected_v2" - val testProps = Seq( Prop("affinity_score", "0.0", DOUBLE), Prop("is_blocked", "false", BOOLEAN), @@ -107,107 +78,43 @@ trait TestCommonWithModels { val hTableTTL = None - def createTestService() = { + def createTestService(ver: String) = { implicit val session = AutoSession - management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV3, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV4, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") + val service = serviceName(ver) + management.createService(service, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") } - def deleteTestService() = { + def deleteTestService(ver: String) = { implicit val session = AutoSession - Management.deleteService(serviceName) - Management.deleteService(serviceNameV2) - Management.deleteService(serviceNameV3) - Management.deleteService(serviceNameV4) + val service = serviceName(ver) + Management.deleteService(service) } - def deleteTestLabel() = { + def deleteTestLabel(ver: String) = { implicit val session = AutoSession - Management.deleteLabel(labelName) - Management.deleteLabel(labelNameV2) - Management.deleteLabel(undirectedLabelName) - Management.deleteLabel(undirectedLabelNameV2) + val label = labelName(ver) + val undirectedLabel = undirectedLabelName(ver) + Management.deleteLabel(label) + Management.deleteLabel(undirectedLabel) } - def createTestLabel() = { + def createTestLabel(ver: String) = { implicit val session = AutoSession - management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType, - isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4") - - management.createLabel(labelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2, - isDirected = true, serviceNameV2, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4") - - management.createLabel(labelNameV3, serviceNameV3, columnNameV3, columnTypeV3, serviceNameV3, tgtColumnNameV3, tgtColumnTypeV3, - isDirected = true, serviceNameV3, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4") - - management.createLabel(labelNameV4, serviceNameV4, columnNameV4, columnTypeV4, serviceNameV4, tgtColumnNameV4, tgtColumnTypeV4, - isDirected = true, serviceNameV4, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4") - - management.createLabel(undirectedLabelName, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType, - isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4") - - management.createLabel(undirectedLabelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2, - isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4") + val label = labelName(ver) + val service = serviceName(ver) + val column = columnName(ver) + management.createLabel(label, service, column, columnType, service, column, columnType, + isDirected = true, service, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, ver, false, "lg4") } - def service = Service.findByName(serviceName, useCache = false).get - - def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get - - def serviceV3 = Service.findByName(serviceNameV3, useCache = false).get - - def serviceV4 = Service.findByName(serviceNameV4, useCache = false).get - - def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get - - def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get - - def columnV3 = ServiceColumn.find(serviceV3.id.get, columnNameV3, useCache = false).get - - def columnV4 = ServiceColumn.find(serviceV4.id.get, columnNameV4, useCache = false).get - - def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get - - def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get - - def tgtColumnV3 = ServiceColumn.find(serviceV3.id.get, tgtColumnNameV3, useCache = false).get - - def tgtColumnV4 = ServiceColumn.find(serviceV4.id.get, tgtColumnNameV4, useCache = false).get - - def label = Label.findByName(labelName, useCache = false).get - - def labelV2 = Label.findByName(labelNameV2, useCache = false).get - - def labelV3 = Label.findByName(labelNameV3, useCache = false).get - - def labelV4 = Label.findByName(labelNameV4, useCache = false).get - - def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get - - def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get - + def service(ver: String) = Service.findByName(serviceName(ver), useCache = false).get + def column(ver: String) = ServiceColumn.find(service(ver).id.get, columnName(ver), useCache = false).get + def tgtColumn(ver: String) = ServiceColumn.find(service(ver).id.get, tgtColumnName(ver), useCache = false).get + def label(ver: String) = Label.findByName(labelName(ver), useCache = false).get + def undirectedLabel(ver: String) = Label.findByName(undirectedLabelName(ver), useCache = false).get def dir = GraphUtil.directions("out") - def op = GraphUtil.operations("insert") - def labelOrderSeq = LabelIndex.DefaultSeq - - def labelWithDir = LabelWithDirection(label.id.get, dir) - - def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir) - - def labelWithDirV3 = LabelWithDirection(labelV3.id.get, dir) - - def labelWithDirV4 = LabelWithDirection(labelV4.id.get, dir) - - def queryParam = QueryParam(labelWithDir) - - def queryParamV2 = QueryParam(labelWithDirV2) - - def queryParamV3 = QueryParam(labelWithDirV3) - - def queryParamV4 = QueryParam(labelWithDirV4) - + def labelWithDir(ver: String) = LabelWithDirection(label(ver).id.get, dir) + def queryParam(ver: String) = QueryParam(labelWithDir(ver)) } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala index 66d84e44..53ac18f3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,131 +19,48 @@ package org.apache.s2graph.core.models -import org.apache.s2graph.core.TestCommonWithModels +import org.apache.s2graph.core.{CommonTest, TestCommonWithModels} import org.apache.s2graph.core.mysqls.Label import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll { override def beforeAll(): Unit = { - initTests() + versions map { v => + val ver = s"v$v" + initTests(ver) + } } override def afterAll(): Unit = { graph.shutdown() } - // val serviceName = "testService" - // val newServiceName = "newTestService" - // val cluster = "localhost" - // val hbaseTableName = "s2graph-dev" - // val columnName = "user_id" - // val columnType = "long" - // val labelName = "model_test_label" - // val newLabelName = "new_model_test_label" - // val columnMetaName = "is_valid_user" - // val labelMetaName = "is_hidden" - // val hbaseTableTTL = -1 - // val id = 1 - // - // val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster, - // "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1)) - // val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get, - // "columnName" -> columnName, "columnType" -> columnType)) - // val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte)) - // val label = HLabel(Map("id" -> id, "label" -> labelName, - // "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType, - // "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType, - // "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get, - // "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1 - // )) - // val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte, - // "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false)) - // val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte, - // "metaSeqs" -> "0", "formular" -> "none")) - test("test Label.findByName") { - val labelOpt = Label.findByName(labelName, useCache = false) - println(labelOpt) - labelOpt.isDefined shouldBe true - val indices = labelOpt.get.indices - indices.size > 0 shouldBe true - println(indices) - val defaultIndexOpt = labelOpt.get.defaultIndex - println(defaultIndexOpt) - defaultIndexOpt.isDefined shouldBe true - val metas = labelOpt.get.metaProps - println(metas) - metas.size > 0 shouldBe true - val srcService = labelOpt.get.srcService - println(srcService) - val tgtService = labelOpt.get.tgtService - println(tgtService) - val service = labelOpt.get.service - println(service) - val srcColumn = labelOpt.get.srcService - println(srcColumn) - val tgtColumn = labelOpt.get.tgtService - println(tgtColumn) + versions map { n => + val ver = s"v$n" + val label = labelName(ver) + test(s"test Label.findByName $ver", CommonTest) { + val labelOpt = Label.findByName(label, useCache = false) + println(labelOpt) + labelOpt.isDefined shouldBe true + val indices = labelOpt.get.indices + indices.size > 0 shouldBe true + println(indices) + val defaultIndexOpt = labelOpt.get.defaultIndex + println(defaultIndexOpt) + defaultIndexOpt.isDefined shouldBe true + val metas = labelOpt.get.metaProps + println(metas) + metas.size > 0 shouldBe true + val srcService = labelOpt.get.srcService + println(srcService) + val tgtService = labelOpt.get.tgtService + println(tgtService) + val service = labelOpt.get.service + println(service) + val srcColumn = labelOpt.get.srcService + println(srcColumn) + val tgtColumn = labelOpt.get.tgtService + println(tgtColumn) + } } - // test("test create") { - // service.create() - // HService.findByName(serviceName, useCache = false) == Some(service) - // - // serviceColumn.create() - // HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn) - // - // columnMeta.create() - // HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta) - // - // label.create() - // HLabel.findByName(labelName, useCache = false) == Some(label) - // - // labelMeta.create() - // HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta) - // - // labelIndex.create() - // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex) - // } - // - // test("test update") { - // service.update("cluster", "...") - // HService.findById(service.id.get, useCache = false).cluster == "..." - // - // service.update("serviceName", newServiceName) - // assert(HService.findByName(serviceName, useCache = false) == None) - // HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get} - // - // label.update("label", newLabelName) - // HLabel.findById(label.id.get, useCache = false).label == "newLabelName" - // - // label.update("consistencyLevel", "strong") - // HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" && - // HLabel.findByName(newLabelName).isDefined && - // HLabel.findByName(labelName) == None - // - // } - // test("test read by index") { - // val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false) - // val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false) - // labels.length == 1 && - // labels.head == label - // idxs.length == 1 && - // idxs.head == labelIndex - // } - // test("test delete") { - //// HLabel.findByName(labelName).foreach { label => - //// label.deleteAll() - //// } - // HLabel.findByName(newLabelName).foreach { label => - // label.deleteAll() - // } - // HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty && - // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty - // - // service.deleteAll() - // } - - // test("test labelIndex") { - // println(HLabelIndex.findByLabelIdAll(1)) - // } - } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala index 4a46e0ed..58309561 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,6 +22,7 @@ package org.apache.s2graph.core.mysqls import java.util.Properties import com.typesafe.config.ConfigFactory +import org.apache.s2graph.core.CommonTest import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} import scalikejdbc._ @@ -47,7 +48,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } - "Experiment" should "find bucket list" in { + "Experiment" should "find bucket list" taggedAs CommonTest in { Experiment.findBy(1, "exp1") should not be empty Experiment.findBy(1, "exp1").foreach { exp => @@ -56,7 +57,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } } - it should "update bucket list after cache ttl time" in { + it should "update bucket list after cache ttl time" taggedAs CommonTest in { Experiment.findBy(1, "exp1").foreach { exp => val bucket = exp.buckets.head bucket.impressionId should equal("imp1") diff --git a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala index 40166eb7..02d34856 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,65 +21,33 @@ package org.apache.s2graph.core.parsers import org.apache.s2graph.core._ import org.apache.s2graph.core.mysqls.{Label, LabelMeta} -import org.apache.s2graph.core.rest.TemplateHelper import org.apache.s2graph.core.types._ import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.Json -class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() +class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels with TestCommon { - // dummy data for dummy edge - initTests() - - import HBaseType.{VERSION1, VERSION2} + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + // dummy data for dummy edge + initTests(ver) - val ts = System.currentTimeMillis() - val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion)) - - def ids(version: String) = { - val colId = if (version == VERSION2) columnV2.id.get else column.id.get - val srcId = SourceVertexId(colId, InnerVal.withLong(1, version)) - val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version)) - - val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version)) - val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version)) - - val srcVertex = Vertex(srcId, ts) - val tgtVertex = Vertex(tgtId, ts) - val srcVertexStr = Vertex(srcIdStr, ts) - val tgtVertexStr = Vertex(tgtIdStr, ts) - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version) - } + val l = label(ver) + val ld = labelWithDir(ver) + val ts = System.currentTimeMillis() + val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, ver)) + val labelMap = Map(l.label -> l) - def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = { - val whereOpt = WhereParser(label).parse(sql) - whereOpt.isSuccess shouldBe true + val (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) = ids(ver) - println("=================================================================") - println(sql) - println(whereOpt.get) - - val ret = whereOpt.get.filter(edge) - if (ret != expected) { - println("==================") - println(s"$whereOpt") - println(s"$edge") - println("==================") - } - ret shouldBe expected - } - - test("check where clause not nested") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause not nested $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) - val f = validate(label)(edge) _ + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) + val f = validate(l, labelMap)(edge) _ /** labelName label is long-long relation */ f(s"_to=${tgtVertex.innerId.toString}")(true) @@ -88,18 +56,14 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f(s"_to=19230495")(false) f(s"_to!=19230495")(true) } - } - test("check where clause nested") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause nested $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) - val f = validate(label)(edge) _ + val f = validate(l,labelMap)(edge) _ // time == 3 f("time >= 3")(true) @@ -117,19 +81,13 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f("(time in ( 1,2,4 ) or weight between 1 and 9) or is_hidden= true")(true) f("(time in (1,2,3) or weight between 1 and 10) and is_hidden =false")(false) } - } - test("check where clause with from/to long") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause with from/to long $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir - val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner) - val lname = if (schemaVer == VERSION2) labelNameV2 else labelName - val f = validate(label)(edge) _ + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) + val f = validate(l, labelMap)(edge) _ f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true) f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true) @@ -137,31 +95,27 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f(s"_from = -1")(false) f(s"_from in (-1, -0.1)")(false) } - } - test("check where clause with parent") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause with parent $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 1, "name" -> "abc") val parentJs = Json.obj("is_hidden" -> false, "is_blocked" -> false, "weight" -> 20, "time" -> 3, "name" -> "a") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val parentPropsInner = Management.toProps(label, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val parentPropsInner = Management.toProps(l, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val grandParentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner) - val parentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner, + val grandParentEdge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, parentPropsInner) + val parentEdge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, parentPropsInner, parentEdges = Seq(EdgeWithScore(grandParentEdge, 1.0))) - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner, + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner, parentEdges = Seq(EdgeWithScore(parentEdge, 1.0))) println(edge.toString) println(parentEdge.toString) println(grandParentEdge.toString) - val f = validate(label)(edge) _ + val f = validate(l, labelMap)(edge) _ // Compare edge's prop(`_from`) with edge's prop(`name`) f("_from = 1")(true) @@ -184,70 +138,54 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f("_parent._parent.weight = weight")(false) f("_parent._parent.weight = _parent.weight")(true) } - } - - test("replace reserved") { - val ts = 0 - import TemplateHelper._ - - calculate(ts, 1, "hour") should be(hour + ts) - calculate(ts, 1, "day") should be(day + ts) - calculate(ts + 10, 1, "HOUR") should be(hour + ts + 10) - calculate(ts + 10, 1, "DAY") should be(day + ts + 10) - - val body = """{ - "day": ${1day}, - "hour": ${1hour}, - "-day": "${-10 day}", - "-hour": ${-10 hour}, - "now": "${now}" - } - """ - - val parsed = replaceVariable(ts, body) - val json = Json.parse(parsed) + // test("time decay") { + // val ts = System.currentTimeMillis() + // + // for { + // i <- (0 until 10) + // } { + // val timeUnit = 60 * 60 + // val diff = i * timeUnit + // val x = TimeDecay(1.0, 0.05, timeUnit) + // println(x.decay(diff)) + // } + // } + } - (json \ "day").as[Long] should be (1 * day + ts) - (json \ "hour").as[Long] should be (1 * hour + ts) - (json \ "-day").as[Long] should be (-10 * day + ts) - (json \ "-hour").as[Long] should be (-10 * hour + ts) + def ids(version: String) = { + val colId = column(version).id.get + val srcId = SourceVertexId(colId, InnerVal.withLong(1, version)) + val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version)) - (json \ "now").as[Long] should be (ts) + val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version)) + val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version)) - val otherBody = """{ - "nextday": "${next_day}", - "3dayago": "${next_day - 3 day}", - "nexthour": "${next_hour}" - }""" + val srcVertex = Vertex(srcId, ts) + val tgtVertex = Vertex(tgtId, ts) + val srcVertexStr = Vertex(srcIdStr, ts) + val tgtVertexStr = Vertex(tgtIdStr, ts) + (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version) + } - val currentTs = System.currentTimeMillis() - val expectedDayTs = currentTs / day * day + day - val expectedHourTs = currentTs / hour * hour + hour - val threeDayAgo = expectedDayTs - 3 * day - val currentTsLs = (1 until 1000).map(currentTs + _) + def validate(l: Label, labelMap: Map[String, Label])(edge: Edge)(sql: String)(expected: Boolean) = { + val whereOpt = WhereParser(l).parse(sql) + whereOpt.isSuccess shouldBe true - currentTsLs.foreach { ts => - val parsed = replaceVariable(ts, otherBody) - val json = Json.parse(parsed) + println("=================================================================") + println(sql) + println(whereOpt.get) - (json \ "nextday").as[Long] should be(expectedDayTs) - (json \ "nexthour").as[Long] should be(expectedHourTs) - (json \ "3dayago").as[Long] should be(threeDayAgo) + val ret = whereOpt.get.filter(edge) + if (ret != expected) { + println("==================") + println(s"$whereOpt") + println(s"$edge") + println("==================") } + ret shouldBe expected } - // test("time decay") { - // val ts = System.currentTimeMillis() - // - // for { - // i <- (0 until 10) - // } { - // val timeUnit = 60 * 60 - // val diff = i * timeUnit - // val x = TimeDecay(1.0, 0.05, timeUnit) - // println(x.decay(diff)) - // } - // } } + diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala index 43293d55..7472f060 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,7 +24,7 @@ import org.scalatest.{FunSuite, Matchers} class AsynchbaseStorageTest extends FunSuite with Matchers { /** need secured cluster */ -// test("test secure cluster connection") { +// test("test secure cluster connection", HBaseTest) { // val config = ConfigFactory.parseMap( // Map( // "hbase.zookeeper.quorum" -> "localhost", diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala index 5edd5b10..277879a9 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,80 +21,70 @@ package org.apache.s2graph.core.storage.hbase import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta} import org.apache.s2graph.core.types._ -import org.apache.s2graph.core.{IndexEdge, TestCommonWithModels, Vertex} +import org.apache.s2graph.core._ import org.scalatest.{FunSuite, Matchers} -class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() +class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels with TestCommon { + versions map { n => + val ver = s"v$n" + initTests(ver) + val l = label(ver) + + /** note that props have to be properly set up for equals */ + test(s"test serializer/deserializer for index edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withLong(101, ver) + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, + 1.toByte -> InnerVal.withDouble(2.1, ver)) + } + + test(s"test serializer/deserializer for degree edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withStr("0", ver) + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map( + LabelMeta.degreeSeq -> InnerVal.withLong(10, ver), + LabelMeta.timeStampSeq -> tsInnerVal) + + check(ver, l, ts, to, props) + } + + test(s"test serializer/deserializer for incrementCount index edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withLong(101, ver) + + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, + 1.toByte -> InnerVal.withDouble(2.1, ver), + LabelMeta.countSeq -> InnerVal.withLong(10, ver)) + + check(ver, l, ts, to, props) + } + } /** * check if storage serializer/deserializer can translate from/to bytes array. - * @param l: label for edge. - * @param ts: timestamp for edge. - * @param to: to VertexId for edge. - * @param props: expected props of edge. + * @param ver: schema version + * @param l: label for edge + * @param ts: timestamp for edge + * @param to: to VertexId for edge + * @param props: expected props of edge */ - def check(l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = { + def check(ver: String, l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = { val from = InnerVal.withLong(1, l.schemaVersion) - val vertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, from) - val tgtVertexId = TargetVertexId(HBaseType.DEFAULT_COL_ID, to) + val vertexId = SourceVertexId(GraphType.DEFAULT_COL_ID, from) + val tgtVertexId = TargetVertexId(GraphType.DEFAULT_COL_ID, to) val vertex = Vertex(vertexId, ts) val tgtVertex = Vertex(tgtVertexId, ts) val labelWithDir = LabelWithDirection(l.id.get, 0) val indexEdge = IndexEdge(vertex, tgtVertex, labelWithDir, 0, ts, LabelIndex.DefaultSeq, props) - val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam, + val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam(ver), graph.storage.indexEdgeSerializer(indexEdge).toKeyValues, l.schemaVersion, None) _indexEdgeOpt should not be empty indexEdge should be(_indexEdgeOpt.get) } - - - /** note that props have to be properly set up for equals */ - test("test serializer/deserializer for index edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withLong(101, l.schemaVersion) - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, - 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion)) - - check(l, ts, to, props) - } - } - - test("test serializer/deserializer for degree edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withStr("0", l.schemaVersion) - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map( - LabelMeta.degreeSeq -> InnerVal.withLong(10, l.schemaVersion), - LabelMeta.timeStampSeq -> tsInnerVal) - - check(l, ts, to, props) - } - } - - test("test serializer/deserializer for incrementCount index edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withLong(101, l.schemaVersion) - - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, - 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion), - LabelMeta.countSeq -> InnerVal.withLong(10, l.schemaVersion)) - - check(l, ts, to, props) - } - } } diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/redis/RedisCrudTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/redis/RedisCrudTest.scala new file mode 100644 index 00000000..d86f8c33 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/redis/RedisCrudTest.scala @@ -0,0 +1,275 @@ +package org.apache.s2graph.core.storage.redis + +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.apache.s2graph.core.Integrate.IntegrateCommon +import org.apache.s2graph.core.mysqls.Label +import org.apache.s2graph.core.rest.RequestParser +import org.apache.s2graph.core.{Graph, Management, RedisTest} +import org.scalatest.BeforeAndAfterEach +import play.api.libs.json.{JsValue, Json} + +import scala.collection.JavaConversions._ +import scala.concurrent.ExecutionContext + +/** + * Created by june.kay on 2016. 1. 20.. + */ +class RedisCrudTest extends IntegrateCommon with BeforeAndAfterEach { + import TestUtil._ + + val insert = "insert" + val increment = "increment" + val e = "e" + val sid1 = 1000001 + val sid2 = 1000002 + val sid3 = 1000003 + val sid4 = 1000004 + val sid6 = 1000006 + val tid1 = 1000 + val tid2 = 1100 + val tid3 = 1110 + val tid4 = 2000 + + val testLabel = "redis_crud_label" + val testService = "redis_crud_service" + val testColumn = "redis_crud_column" + + override def beforeAll = { + config = ConfigFactory.load() + .withValue("storage.engine", ConfigValueFactory.fromAnyRef("redis")) // for redis test + .withValue("storage.redis.instances", ConfigValueFactory.fromIterable(List[String]("localhost"))) // for redis test + +// println(s">> Config for storage.engine : ${config.getString("storage.engine")}") +// println(s">> Config for redis.instances : ${config.getStringList("storage.redis.instances").mkString(",")}") + + graph = new Graph(config)(ExecutionContext.Implicits.global) + parser = new RequestParser(graph.config) + management = new Management(graph) + initTestData() + } + + /** + * Make Service, Label, Vertex for integrate test + */ + override def initTestData() = { +// println("[Redis init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + Management.deleteService(testService) + + // 1. createService + val jsValue = Json.parse(s"""{"serviceName" : "$testService"}""") + val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = + parser.toServiceElements(jsValue) + + val tryRes = + management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) +// println(s">> Service created : $createService, $tryRes") + + def lc(ver: String, consistency: String = "strong") = { + s""" + { + "label": "$testLabel", + "srcServiceName": "$testService", + "srcColumnName": "$testColumn", + "srcColumnType": "long", + "tgtServiceName": "$testService", + "tgtColumnName": "$testColumn", + "tgtColumnType": "long", + "indices": [ + {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, + {"name": "$index2", "propNames": ["_timestamp"]} + ], + "props": [ + { + "name": "time", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "weight", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "is_hidden", + "dataType": "boolean", + "defaultValue": false + }, + { + "name": "is_blocked", + "dataType": "boolean", + "defaultValue": false + } + ], + "consistencyLevel": "$consistency", + "schemaVersion": "$ver", + "compressionAlgorithm": "gz" + }""" + } + + // with only v4 label + val labelNames = Map(testLabel -> lc("v4")) + + for { + (labelName, create) <- labelNames + } { + Management.deleteLabel(labelName) + Label.findByName(labelName, useCache = false) match { + case None => + val json = Json.parse(create) + val tryRes = for { + labelArgs <- parser.toLabelElements(json) + label <- (management.createLabel _).tupled(labelArgs) + } yield label + + tryRes.get + case Some(label) => +// println(s">> Label already exist: $create, $label") + } + } + + val vertexPropsKeys = List("age" -> "int") + + vertexPropsKeys.map { case (key, keyType) => + Management.addVertexProp(testService, testColumn, key, keyType) + } + +// println("[Redis init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + } + + + test("test insert/check/get edges", RedisTest) { + insertEdgesSync( + toEdge(1, insert, e, sid1, tid1, testLabel), + toEdge(1, insert, e, sid1, tid2, testLabel), + toEdge(1, insert, e, sid1, tid3, testLabel), + toEdge(1, insert, e, sid2, tid4, testLabel) + ) + def queryCheckEdges(fromId: Int, toId: Int): JsValue = Json.parse( + s""" + |[{ + | "label": "$testLabel", + | "direction": "out", + | "from": $fromId, + | "to": $toId + |}] + """.stripMargin + ) + def query(id: Int, label: String, offset: Int = 0, limit: Int = 100) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testService", + "columnName": "$testColumn", + "id": $id + }], + "steps": [ + [ { + "label": "$label", + "direction": "out", + "offset": $offset, + "limit": $limit + } + ]] + } + """) + + + var result = checkEdgesSync(queryCheckEdges(sid1, tid1)) + println(result.toString()) + (result \ "size").toString should be("1") // edge 1000001 -> 1000 should be present + + result = checkEdgesSync(queryCheckEdges(sid2, tid4)) + println(result.toString()) + (result \ "size").toString should be("1") // edge 1000002 -> 2000 should be present + + result = getEdgesSync(query(sid1, testLabel, 0, 10)) + println(result.toString()) + (result \ "size").toString should be("3") // edge 1000001 -> 1000, 1100, 1110 should be present + } + + test("get vertex", RedisTest) { + val ids = Array(sid1, sid2) + val q = vertexQueryJson(testService, testColumn, ids) + + val rs = getVerticesSync(q) + rs.as[Array[JsValue]].size should be (2) + } + +// test("insert vertex", RedisTest) { +// val ids = (sid3 until sid6) +// val data = vertexInsertsPayload(testServiceName, testColumnName, ids) +// val payload = Json.parse(Json.toJson(data).toString()) +// println(Json.prettyPrint(payload)) +// +// val vertices = parser.toVertices(payload, "insert", Option(testServiceName), Option(testColumnName)) +// Await.result(graph.mutateVertices(vertices, true), HttpRequestWaitingTime) +// +// +// val q = vertexQueryJson(testServiceName, testColumnName, ids) +// println("vertex get query: " + q.toString()) +// +// val rs = getVerticesSync(q) +// println("vertex get result: " + rs.toString()) +// rs.as[Array[JsValue]].size should be (3) +// } +// +// test("test increment", RedisTest) { +// def queryTo(id: Int, to:Int, offset: Int = 0, limit: Int = 10) = Json.parse( +// s""" +// |{ +// | "srcVertices": [{ +// | "serviceName": "$testServiceName", +// | "columnName": "$testColumnName", +// | "id": $id +// | }], +// | "steps": [ +// | [{ +// | "label": "$testLabelNameV4", +// | "direction": "out", +// | "offset": $offset, +// | "limit": $limit, +// | "where": "_to=$to" +// | }] +// | ] +// |} +// """.stripMargin +// ) +// +// val incrementVal = 10 +// mutateEdgesSync( +// toEdge(1, increment, e, sid3, sid4, testLabelNameV4, "{\"weight\":%s}".format(incrementVal), "out") +// ) +// +// val resp = getEdgesSync(queryTo(sid3, sid4)) +// println(s"Result: ${Json.prettyPrint(resp)}") +// (resp \ "size").toString should be ("1") // edge 1000003 -> 1000004 should be present +// +// val result = (resp \\ "results" ).head(0) +// (result \ "props" \ "weight" ).toString should be (s"$incrementVal") +// } +// +// test("deleteAll", RedisTest) { +// val deletedAt = 100 +// var result = getEdgesSync(querySingle(sid1, testLabelNameV4, 0, 10)) +// +// println(s"before deleteAll: ${Json.prettyPrint(result)}") +// +// result = getEdgesSync(querySingle(sid1, testLabelNameV4, 0, 10)) +// println(result.toString()) +// (result \ "size").toString should be("3") // edge 1000001 -> 1000, 1100, 1110 should be present +// +// val deleteParam = Json.arr( +// Json.obj("label" -> testLabelNameV4, +// "direction" -> "out", +// "ids" -> Json.arr(sid1.toString), +// "timestamp" -> deletedAt +// ) +// ) +// deleteAllSync(deleteParam) +// +// result = getEdgesSync(querySingle(sid1, testLabelNameV4, 0, 10)) +// println(result.toString()) +// (result \ "size").toString should be("0") // edge 1000001 -> 1000, 1100, 1110 should be deleted +// +// } + +} diff --git a/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala index 2f6d0664..91adf647 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,128 +20,132 @@ package org.apache.s2graph.core.types import org.apache.hadoop.hbase.util.Bytes -import org.apache.s2graph.core.TestCommonWithModels +import org.apache.s2graph.core.{CommonTest, TestCommonWithModels} import org.scalatest.{FunSuite, Matchers} class InnerValTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() - import HBaseType.VERSION2 - val decimals = List( - BigDecimal(Long.MinValue), - BigDecimal(Int.MinValue), - BigDecimal(Double.MinValue), - BigDecimal(Float.MinValue), - BigDecimal(Short.MinValue), - BigDecimal(Byte.MinValue), + versions map { n => + val ver = s"v$n" + initTests(ver) - BigDecimal(-1), - BigDecimal(0), - BigDecimal(1), - BigDecimal(Long.MaxValue), - BigDecimal(Int.MaxValue), - BigDecimal(Double.MaxValue), - BigDecimal(Float.MaxValue), - BigDecimal(Short.MaxValue), - BigDecimal(Byte.MaxValue) - ) - val booleans = List( - false, true - ) - val strings = List( - "abc", "abd", "ac", "aca" - ) - val texts = List( - (0 until 1000).map(x => "a").mkString - ) - val blobs = List( - (0 until 1000).map(x => Byte.MaxValue).toArray - ) - def testEncodeDecode(ranges: List[InnerValLike], version: String) = { - for { - innerVal <- ranges - } { - val bytes = innerVal.bytes - val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version) - innerVal == decoded shouldBe true - bytes.length == numOfBytesUsed shouldBe true + + val decimals = List( + BigDecimal(Long.MinValue), + BigDecimal(Int.MinValue), + BigDecimal(Double.MinValue), + BigDecimal(Float.MinValue), + BigDecimal(Short.MinValue), + BigDecimal(Byte.MinValue), + + BigDecimal(-1), + BigDecimal(0), + BigDecimal(1), + BigDecimal(Long.MaxValue), + BigDecimal(Int.MaxValue), + BigDecimal(Double.MaxValue), + BigDecimal(Float.MaxValue), + BigDecimal(Short.MaxValue), + BigDecimal(Byte.MaxValue) + ) + val booleans = List( + false, true + ) + val strings = List( + "abc", "abd", "ac", "aca" + ) + val texts = List( + (0 until 1000).map(x => "a").mkString + ) + val blobs = List( + (0 until 1000).map(x => Byte.MaxValue).toArray + ) + def testEncodeDecode(ranges: List[InnerValLike], version: String) = { + for { + innerVal <- ranges + } { + val bytes = innerVal.bytes + val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version) + innerVal == decoded shouldBe true + bytes.length == numOfBytesUsed shouldBe true + } } - } - // test("big decimal") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)} - // testEncodeDecode(innerVals, version) - // } - // } - // test("text") { - // for { - // version <- List(VERSION2) - // } { - // val innerVals = texts.map { t => InnerVal.withStr(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("string") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = strings.map { t => InnerVal.withStr(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("blob") { - // for { - // version <- List(VERSION2) - // } { - // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("boolean") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - test("korean") { - val small = InnerVal.withStr("가", VERSION2) - val large = InnerVal.withStr("나", VERSION2) - val smallBytes = small.bytes - val largeBytes = large.bytes + // test("big decimal") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)} + // testEncodeDecode(innerVals, version) + // } + // } + // test("text") { + // for { + // version <- List(VERSION2) + // } { + // val innerVals = texts.map { t => InnerVal.withStr(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("string") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = strings.map { t => InnerVal.withStr(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("blob") { + // for { + // version <- List(VERSION2) + // } { + // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("boolean") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + test(s"korean $ver", CommonTest) { + val small = InnerVal.withStr("가", ver) + val large = InnerVal.withStr("나", ver) + val smallBytes = small.bytes + val largeBytes = large.bytes - println (Bytes.compareTo(smallBytes, largeBytes)) - true + println (Bytes.compareTo(smallBytes, largeBytes)) + true + } + // test("innerVal") { + // val srcVal = InnerVal.withLong(44391298, VERSION2) + // val srcValV1 = InnerVal.withLong(44391298, VERSION1) + // val tgtVal = InnerVal.withLong(7295564, VERSION2) + // + // val a = VertexId(0, srcVal) + // val b = SourceVertexId(0, srcVal) + // val c = TargetVertexId(0, srcVal) + // val aa = VertexId(0, srcValV1) + // val bb = SourceVertexId(0, srcValV1) + // val cc = TargetVertexId(0, srcValV1) + // println(a.bytes.toList) + // println(b.bytes.toList) + // println(c.bytes.toList) + // + // println(aa.bytes.toList) + // println(bb.bytes.toList) + // println(cc.bytes.toList) + // } + // test("aa") { + // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes + // val pbr = new SimplePositionedByteRange(bytes) + // pbr.setOffset(1) + // println(pbr.getPosition) + // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr) + // println(pbr.getPosition) + // true + // } } - // test("innerVal") { - // val srcVal = InnerVal.withLong(44391298, VERSION2) - // val srcValV1 = InnerVal.withLong(44391298, VERSION1) - // val tgtVal = InnerVal.withLong(7295564, VERSION2) - // - // val a = VertexId(0, srcVal) - // val b = SourceVertexId(0, srcVal) - // val c = TargetVertexId(0, srcVal) - // val aa = VertexId(0, srcValV1) - // val bb = SourceVertexId(0, srcValV1) - // val cc = TargetVertexId(0, srcValV1) - // println(a.bytes.toList) - // println(b.bytes.toList) - // println(c.bytes.toList) - // - // println(aa.bytes.toList) - // println(bb.bytes.toList) - // println(cc.bytes.toList) - // } - // test("aa") { - // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes - // val pbr = new SimplePositionedByteRange(bytes) - // pbr.setOffset(1) - // println(pbr.getPosition) - // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr) - // println(pbr.getPosition) - // true - // } } diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala index 75026fa4..961024c6 100644 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala +++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala @@ -21,7 +21,7 @@ package org.apache.s2graph.rest.play.benchmark import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.types.{HBaseType, InnerVal, SourceVertexId} +import org.apache.s2graph.core.types.{GraphType, InnerVal, SourceVertexId} import play.api.test.{FakeApplication, PlaySpecification} import scala.collection.mutable @@ -66,7 +66,7 @@ class GraphUtilSpec extends BenchmarkCommon with PlaySpecification { "test murmur hash skew2" in { running(FakeApplication()) { - import HBaseType._ + import GraphType._ val testNum = 1000000L val regionCount = 40 val window = Int.MaxValue / regionCount @@ -86,7 +86,7 @@ class GraphUtilSpec extends BenchmarkCommon with PlaySpecification { stats += (0 -> (rangeBytes.head -> 0L)) for (i <- (0L until testNum)) { - val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION)) + val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, GraphType.DEFAULT_VERSION)) val bytes = vertexId.bytes val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString()) val shortVal = counts.getOrElse(shortKey, 0L) + 1L