diff --git a/build.sbt b/build.sbt index 85711636..aea9192e 100755 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,15 @@ lazy val commonSettings = Seq( version := "0.12.1-SNAPSHOT", scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation", "-feature", "-Xlint"), javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map { case (key, value) => "-D" + key + "=" + value }.toSeq, - testOptions in Test += Tests.Argument("-oDF"), + testOptions in Test ++= Seq( + Tests.Argument("-oDF"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "CommonTest"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V1Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V2Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V3Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "V4Test"), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "HBaseTest") + ), parallelExecution in Test := false, resolvers ++= Seq( Resolver.mavenLocal, diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala index 0898ffa4..e917859f 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala @@ -44,4 +44,6 @@ object GraphExceptions { case class FetchTimeoutException(msg: String) extends Exception(msg) case class DropRequestException(msg: String) extends Exception(msg) + + case class UnsupportedVersionException(msg: String) extends Exception(msg) } diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala index 55b3e79f..164115f1 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala @@ -217,7 +217,7 @@ class RestHandler(graph: Graph)(implicit ec: ExecutionContext) { } } - private def getVertices(jsValue: JsValue) = { + def getVertices(jsValue: JsValue) = { val jsonQuery = jsValue val ts = System.currentTimeMillis() val props = "{}" diff --git a/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala new file mode 100644 index 00000000..738c4f90 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/CommonTest.scala @@ -0,0 +1,13 @@ +package org.apache.s2graph.core + +import org.scalatest.Tag + +/** + * @author Junki Kim (wishoping@gmail.com), Hyunsung Jo (hyunsung.jo@gmail.com) on 2016/Feb/23. + */ +object CommonTest extends Tag("CommonTest") +object V1Test extends Tag("V1Test") +object V2Test extends Tag("V2Test") +object V3Test extends Tag("V3Test") +object V4Test extends Tag("V4Test") +object HBaseTest extends Tag("HBaseTest") diff --git a/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala index a018c015..822b1cff 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala @@ -25,530 +25,140 @@ import org.apache.s2graph.core.utils.logger import org.scalatest.FunSuite class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels { - initTests() - - test("toLogString") { - val testLabelName = labelNameV2 - val bulkQueries = List( - ("1445240543366", "update", "{\"is_blocked\":true}"), - ("1445240543362", "insert", "{\"is_hidden\":false}"), - ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), - ("1445240543363", "delete", "{}"), - ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) - - val (srcId, tgtId, labelName) = ("1", "2", testLabelName) - - val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { - Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString - }).mkString("\n") - - val expected = Seq( - Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"), - Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"), - Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"), - Seq("1445240543363", "delete", "e", "1", "2", testLabelName), - Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}") - ).map(_.mkString("\t")).mkString("\n") - - assert(bulkEdge === expected) - } - - test("buildOperation") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - - val newPropsWithTs = Map( - timestampProp, - 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1) - ) - - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isDefined) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: snapshotEdge: None with newProps") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - val newPropsWithTs = Map( - timestampProp, - 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1) - ) + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + initTests(ver) - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isDefined) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - - val snapshotEdge = None - val propsWithTs = Map(timestampProp) - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - - val newPropsWithTs = propsWithTs - - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.isEmpty) - assert(edgeMutate.edgesToInsert.isEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - - test("buildMutation: All props older than snapshotEdge's LastDeletedAt") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - val oldPropsWithTs = Map( - timestampProp, - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val propsWithTs = Map( - timestampProp, - 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 2), - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val snapshotEdge = - Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) - - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.nonEmpty) - assert(edgeMutate.edgesToInsert.isEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) - } - test("buildMutation: All props newer than snapshotEdge's LastDeletedAt") { - val schemaVersion = "v2" - val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion)) - val srcVertex = Vertex(vertexId) - val tgtVertex = srcVertex - - val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1) - val oldPropsWithTs = Map( - timestampProp, - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val propsWithTs = Map( - timestampProp, - 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 4), - LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3) - ) - - val snapshotEdge = - Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) - - val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs) - val newVersion = 0L - val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) - logger.info(edgeMutate.toLogString) - - assert(edgeMutate.newSnapshotEdge.nonEmpty) - assert(edgeMutate.edgesToInsert.nonEmpty) - assert(edgeMutate.edgesToDelete.isEmpty) + val label = labelName(ver) + val dirLabel = labelWithDir(ver) + test(s"toLogString $ver", tag) { + val bulkQueries = List( + ("1445240543366", "update", "{\"is_blocked\":true}"), + ("1445240543362", "insert", "{\"is_hidden\":false}"), + ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), + ("1445240543363", "delete", "{}"), + ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) + + val (srcId, tgtId, labelName) = ("1", "2", label) + + val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { + Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString + }).mkString("\n") + + val expected = Seq( + Seq("1445240543366", "update", "e", "1", "2", label, "{\"is_blocked\":true}"), + Seq("1445240543362", "insert", "e", "1", "2", label, "{\"is_hidden\":false}"), + Seq("1445240543364", "insert", "e", "1", "2", label, "{\"is_hidden\":false,\"weight\":10}"), + Seq("1445240543363", "delete", "e", "1", "2", label), + Seq("1445240543365", "update", "e", "1", "2", label, "{\"time\":1,\"weight\":-10}") + ).map(_.mkString("\t")).mkString("\n") + + assert(bulkEdge === expected) + } + + test(s"buildOperation $ver", tag) { + val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) + val srcVertex = Vertex(vertexId) + val tgtVertex = srcVertex + + val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) + + val snapshotEdge = None + val propsWithTs = Map(timestampProp) + val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) + val newVersion = 0L + + val newPropsWithTs = Map( + timestampProp, + 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, ver), 1) + ) + + val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) + logger.info(edgeMutate.toLogString) + + assert(edgeMutate.newSnapshotEdge.isDefined) + assert(edgeMutate.edgesToInsert.nonEmpty) + assert(edgeMutate.edgesToDelete.isEmpty) + } + + test(s"buildMutation: snapshotEdge: None with newProps $ver", tag) { + val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) + val srcVertex = Vertex(vertexId) + val tgtVertex = srcVertex + + val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) + + val snapshotEdge = None + val propsWithTs = Map(timestampProp) + val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) + val newVersion = 0L + + val newPropsWithTs = Map( + timestampProp, + 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, ver), 1) + ) + + val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) + logger.info(edgeMutate.toLogString) + + assert(edgeMutate.newSnapshotEdge.isDefined) + assert(edgeMutate.edgesToInsert.nonEmpty) + assert(edgeMutate.edgesToDelete.isEmpty) + } + + test(s"buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests $ver", tag) { + val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) + val srcVertex = Vertex(vertexId) + val tgtVertex = srcVertex + + val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) + + val snapshotEdge = None + val propsWithTs = Map(timestampProp) + val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) + val newVersion = 0L + + val newPropsWithTs = propsWithTs + + val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs) + logger.info(edgeMutate.toLogString) + + assert(edgeMutate.newSnapshotEdge.isEmpty) + assert(edgeMutate.edgesToInsert.isEmpty) + assert(edgeMutate.edgesToDelete.isEmpty) + } + + test(s"buildMutation: All props older than snapshotEdge's LastDeletedAt $ver", tag) { + val vertexId = VertexId(0, InnerVal.withStr("dummy", ver)) + val srcVertex = Vertex(vertexId) + val tgtVertex = srcVertex + + val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 1) + val oldPropsWithTs = Map( + timestampProp, + LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) + ) + + val propsWithTs = Map( + timestampProp, + 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 2), + LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, ver), 3) + ) + + val snapshotEdge = + Option(Edge(srcVertex, tgtVertex, dirLabel, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs)) + + val requestEdge = Edge(srcVertex, tgtVertex, dirLabel, propsWithTs = propsWithTs) + val newVersion = 0L + val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs) + logger.info(edgeMutate.toLogString) + + assert(edgeMutate.newSnapshotEdge.nonEmpty) + assert(edgeMutate.edgesToInsert.isEmpty) + assert(edgeMutate.edgesToDelete.isEmpty) + } } -} - -//import com.kakao.s2graph.core.types._ -//import org.hbase.async.PutRequest -//import org.scalatest.{FunSuite, Matchers} -// -///** -// * Created by shon on 5/29/15. -// */ -//class EdgeTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels { -// -// -// import HBaseType.{VERSION1, VERSION2} -// -// -// def srcVertex(innerVal: InnerValLike)(version: String) = { -// val colId = if (version == VERSION1) column.id.get else columnV2.id.get -// Vertex(SourceVertexId(colId, innerVal), ts) -// } -// -// def tgtVertex(innerVal: InnerValLike)(version: String) = { -// val colId = if (version == VERSION1) column.id.get else columnV2.id.get -// Vertex(TargetVertexId(colId, innerVal), ts) -// } -// -// -// def testEdges(version: String) = { -// val innerVals = if (version == VERSION1) intInnerVals else intInnerValsV2 -// val tgtV = tgtVertex(innerVals.head)(version) -// innerVals.tail.map { intInnerVal => -// val labelWithDirection = if (version == VERSION1) labelWithDir else labelWithDirV2 -// val idxPropsList = if (version == VERSION1) idxPropsLs else idxPropsLsV2 -// idxPropsList.map { idxProps => -// val currentTs = idxProps.toMap.get(0.toByte).get.toString.toLong -// val idxPropsWithTs = idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, currentTs) } -// -// Edge(srcVertex(intInnerVal)(version), tgtV, labelWithDirection, op, currentTs, currentTs, idxPropsWithTs.toMap) -// } -// } -// } -// -// def testPropsUpdate(oldProps: Map[Byte, InnerValLikeWithTs], -// newProps: Map[Byte, InnerValLikeWithTs], -// expected: Map[Byte, Any], -// expectedShouldUpdate: Boolean) -// (f: PropsPairWithTs => (Map[Byte, InnerValLikeWithTs], Boolean))(version: String) = { -// -// val timestamp = newProps.toList.head._2.ts -// val (updated, shouldUpdate) = f((oldProps, newProps, timestamp, version)) -// val rets = for { -// (k, v) <- expected -// } yield { -// v match { -// case v: String => -// v match { -// case "left" => updated.get(k).isDefined && updated(k) == oldProps(k) -// case "right" => updated.get(k).isDefined && updated(k) == newProps(k) -// case "none" => updated.get(k).isEmpty -// } -// case value: InnerValLikeWithTs => updated.get(k).get == value -// case _ => throw new RuntimeException(s"not supported keyword: $v") -// } -// } -// println(rets) -// rets.forall(x => x) && shouldUpdate == expectedShouldUpdate -// } -// -// def testEdgeWithIndex(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = { -// val rets = for { -// edgeForSameTgtVertex <- edges -// } yield { -// val head = edgeForSameTgtVertex.head -// val start = head -// var prev = head -// val rets = for { -// edge <- edgeForSameTgtVertex.tail -// } yield { -// println(s"prevEdge: $prev") -// println(s"currentEdge: $edge") -// val prevEdgeWithIndex = edge.edgesWithIndex -// val edgesWithIndex = edge.edgesWithIndex -// -// /** test encodeing decoding */ -// for { -// edgeWithIndex <- edge.edgesWithIndex -// put <- edgeWithIndex.buildPutsAsync() -// kv <- putToKeyValues(put.asInstanceOf[PutRequest]) -// } { -// val decoded = Edge.toEdge(kv, queryParam, None, Seq()) -// val comp = decoded.isDefined && decoded.get == edge -// println(s"${decoded.get}") -// println(s"$edge") -// println(s"${decoded.get == edge}") -// comp shouldBe true -// -// /** test order -// * same source, target vertex. same indexProps keys. -// * only difference is indexProps values so comparing qualifier is good enough -// * */ -// for { -// prevEdgeWithIndex <- prev.edgesWithIndex -// } { -// println(edgeWithIndex.qualifier) -// println(prevEdgeWithIndex.qualifier) -// println(edgeWithIndex.qualifier.bytes.toList) -// println(prevEdgeWithIndex.qualifier.bytes.toList) -// /** since index of this test label only use 0, 1 as indexProps -// * if 0, 1 is not different then qualifier bytes should be same -// * */ -// val comp = lessThanEqual(edgeWithIndex.qualifier.bytes, prevEdgeWithIndex.qualifier.bytes) -// comp shouldBe true -// } -// } -// prev = edge -// } -// } -// } -// -// def testInvertedEdge(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = { -// val rets = for { -// edgeForSameTgtVertex <- edges -// } yield { -// val head = edgeForSameTgtVertex.head -// val start = head -// var prev = head -// val rets = for { -// edge <- edgeForSameTgtVertex.tail -// } yield { -// println(s"prevEdge: $prev") -// println(s"currentEdge: $edge") -// val prevEdgeWithIndexInverted = prev.toSnapshotEdge -// val edgeWithInvertedIndex = edge.toSnapshotEdge -// /** test encode decoding */ -// -// val put = edgeWithInvertedIndex.buildPutAsync() -// for { -// kv <- putToKeyValues(put) -// } yield { -// val decoded = Edge.toSnapshotEdge(kv, queryParam, None, isInnerCall = false, Seq()) -// val comp = decoded.isDefined && decoded.get == edge -// println(s"${decoded.get}") -// println(s"$edge") -// println(s"${decoded.get == edge}") -// comp shouldBe true -// -// /** no need to test ordering because qualifier only use targetVertexId */ -// } -// prev = edge -// } -// } -// } -// -// test("insert for edgesWithIndex version 2") { -// val version = VERSION2 -// testEdgeWithIndex(testEdges(version))(queryParamV2) -// } -// test("insert for edgesWithIndex version 1") { -// val version = VERSION1 -// testEdgeWithIndex(testEdges(version))(queryParam) -// } -// -// test("insert for edgeWithInvertedIndex version 1") { -// val version = VERSION1 -// testInvertedEdge(testEdges(version))(queryParam) -// } -// -// test("insert for edgeWithInvertedIndex version 2") { -// val version = VERSION2 -// testInvertedEdge(testEdges(version))(queryParamV2) -// } -// -// -// // /** test cases for each operation */ -// -// def oldProps(timestamp: Long, version: String) = { -// Map( -// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp - 2, timestamp - 2, version), -// 1.toByte -> InnerValLikeWithTs.withLong(0L, timestamp, version), -// 2.toByte -> InnerValLikeWithTs.withLong(1L, timestamp - 1, version), -// 4.toByte -> InnerValLikeWithTs.withStr("old", timestamp - 1, version) -// ) -// } -// -// def newProps(timestamp: Long, version: String) = { -// Map( -// 2.toByte -> InnerValLikeWithTs.withLong(-10L, timestamp, version), -// 3.toByte -> InnerValLikeWithTs.withLong(20L, timestamp, version) -// ) -// } -// -// def deleteProps(timestamp: Long, version: String) = Map( -// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp, timestamp, version) -// ) -// -// /** upsert */ -// test("Edge.buildUpsert") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "none", -// 2.toByte -> "right", -// 3.toByte -> "right", -// 4.toByte -> "none") -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true -// } -// test("Edge.buildUpsert shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 3.toByte -> "none", -// 4.toByte -> "left") -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true -// } -// -// /** update */ -// test("Edge.buildUpdate") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "right", -// 3.toByte -> "right", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true -// } -// test("Edge.buildUpdate shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 3.toByte -> "none", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true -// } -// -// /** delete */ -// test("Edge.buildDelete") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2) -// val newState = deleteProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "right", -// 1.toByte -> "none", -// 2.toByte -> "none", -// 4.toByte -> "none" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true -// } -// test("Edge.buildDelete shouldUpdate false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2) -// val newState = deleteProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left", -// 4.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true -// } -// -// /** increment */ -// test("Edge.buildIncrement") { -// val shouldUpdate = true -// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte) -// val newState = newProps(ts + 1, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> InnerValLikeWithTs.withLong(-9L, ts - 1, VERSION2), -// 3.toByte -> "right" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true -// } -// test("Edge.buildIncrement shouldRepalce false") { -// val shouldUpdate = false -// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte) -// val newState = newProps(ts - 10, VERSION2) -// val expected = Map( -// labelMeta.lastDeletedAt -> "left", -// 1.toByte -> "left", -// 2.toByte -> "left" -// ) -// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true -// } -// -// test("Edge`s srcVertex") { -// -// val version = VERSION2 -// val srcId = InnerVal.withLong(10, version) -// val tgtId = InnerVal.withStr("abc", version) -// val srcColumn = columnV2 -// val tgtColumn = tgtColumnV2 -// val srcVertexId = VertexId(srcColumn.id.get, srcId) -// val tgtVertexId = VertexId(tgtColumn.id.get, tgtId) -// -// val srcVertex = Vertex(srcVertexId) -// val tgtVertex = Vertex(tgtVertexId) -// -// val labelId = undirectedLabelV2.id.get -// -// val outDir = LabelWithDirection(labelId, GraphUtil.directions("out")) -// val inDir = LabelWithDirection(labelId, GraphUtil.directions("in")) -// val bothDir = LabelWithDirection(labelId, GraphUtil.directions("undirected")) -// -// val op = GraphUtil.operations("insert") -// -// -// val bothEdge = Edge(srcVertex, tgtVertex, bothDir) -// println(s"edge: $bothEdge") -// bothEdge.relatedEdges.foreach { edge => -// println(edge) -// } -// -// } -// test("edge buildIncrementBulk") { -// -// /** -// * 172567371 List(97, 74, 2, 117, -74, -44, -76, 0, 0, 4, 8, 2) -//169116518 List(68, -110, 2, 117, -21, 124, -103, 0, 0, 4, 9, 2) -//11646834 List(17, 33, 2, 127, 78, 72, -115, 0, 0, 4, 9, 2) -//148171217 List(62, 54, 2, 119, 43, 22, 46, 0, 0, 4, 9, 2) -//116315188 List(41, 86, 2, 121, 17, 43, -53, 0, 0, 4, 9, 2) -//180667876 List(48, -82, 2, 117, 59, 58, 27, 0, 0, 4, 8, 2) -//4594410 List(82, 29, 2, 127, -71, -27, 21, 0, 0, 4, 8, 2) -//151435444 List(1, 105, 2, 118, -7, 71, 75, 0, 0, 4, 8, 2) -//168460895 List(67, -35, 2, 117, -11, 125, -96, 0, 0, 4, 9, 2) -//7941614 List(115, 67, 2, 127, -122, -46, 17, 0, 0, 4, 8, 2) -//171169732 List(61, -42, 2, 117, -52, 40, 59, 0, 0, 4, 9, 2) -//174381375 List(91, 2, 2, 117, -101, 38, -64, 0, 0, 4, 9, 2) -//12754019 List(9, -80, 2, 127, 61, 99, -100, 0, 0, 4, 9, 2) -//175518092 List(111, 32, 2, 117, -119, -50, 115, 0, 0, 4, 8, 2) -//174748531 List(28, -81, 2, 117, -107, -116, -116, 0, 0, 4, 8, 2) -// -// */ -// // val incrementsOpt = Edge.buildIncrementDegreeBulk("169116518", "talk_friend_long_term_agg_test", "out", 10) -// // -// // for { -// // increments <- incrementsOpt -// // increment <- increments -// // (cf, qs) <- increment.getFamilyMapOfLongs -// // (q, v) <- qs -// // } { -// // println(increment.getRow.toList) -// // println(q.toList) -// // println(v) -// // } -// //List(68, -110, -29, -4, 116, -24, 124, -37, 0, 0, 52, -44, 2) -// } -//} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala index 3911e0d2..9aeced64 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,158 +26,188 @@ class CrudTest extends IntegrateCommon { import CrudHelper._ import TestUtil._ - test("test CRUD") { - var tcNum = 0 - var tcString = "" - var bulkQueries = List.empty[(Long, String, String)] - var expected = Map.empty[String, String] - - val curTime = System.currentTimeMillis - val t1 = curTime + 0 - val t2 = curTime + 1 - val t3 = curTime + 2 - val t4 = curTime + 3 - val t5 = curTime + 4 - - val tcRunner = new CrudTestRunner() - tcNum = 1 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 2 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 3 - tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 4 - tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 5 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" - bulkQueries = List( - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 6 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " - bulkQueries = List( - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 7 - tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 8 - tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 9 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " - bulkQueries = List( - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 10 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" - bulkQueries = List( - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 11 - tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - tcNum = 12 - tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - - tcNum = 13 - tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " - bulkQueries = List( - (t5, "update", "{\"is_blocked\": true}"), - (t1, "insert", "{\"is_hidden\": false}"), - (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), - (t2, "delete", ""), - (t4, "update", "{\"time\": 1, \"weight\": -10}")) - expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") - - tcRunner.run(tcNum, tcString, bulkQueries, expected) - } + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + + test(s"test CRUD $ver", tag) { + val tcRunner = new CrudTestRunner() + tcRunner.test(ver) + } + // val label = getLabelName(ver) + // test(s"interval $ver", tag) { + // + // var edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index + // (edges \ "size").toString should be("1") + // + // edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index + // (edges \ "size").toString should be("2") + // + // edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 11)) // test interval on weight index + // (edges \ "size").toString should be("1") + // + // edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 20)) // test interval on weight index + // (edges \ "size").toString should be("2") + // } + } object CrudHelper { class CrudTestRunner { var seed = 0 - def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { + def test(ver: String) = { + var tcNum = 0 + var tcString = "" + var bulkQueries = List.empty[(Long, String, String)] + var expected = Map.empty[String, String] + + val curTime = System.currentTimeMillis + val t1 = curTime + 0 + val t2 = curTime + 1 + val t3 = curTime + 2 + val t4 = curTime + 3 + val t5 = curTime + 4 + + tcNum = 1 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 2 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 3 + tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 4 + tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 5 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" + bulkQueries = List( + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 6 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " + bulkQueries = List( + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 7 + tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 8 + tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 9 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " + bulkQueries = List( + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 10 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" + bulkQueries = List( + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 11 + tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 12 + tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + run(ver, tcNum, tcString, bulkQueries, expected) + + tcNum = 13 + tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " + bulkQueries = List( + (t5, "update", "{\"is_blocked\": true}"), + (t1, "insert", "{\"is_hidden\": false}"), + (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), + (t2, "delete", ""), + (t4, "update", "{\"time\": 1, \"weight\": -10}")) + expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") + + run(ver, tcNum, tcString, bulkQueries, expected) + } + + def run(ver: String, tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { + val labelName = getLabelName(ver) + for { - labelName <- List(testLabelName, testLabelName2) i <- 0 until NumOfEachTest } { seed += 1 @@ -188,9 +218,9 @@ class CrudTest extends IntegrateCommon { /** insert edges */ println(s"---- TC${tcNum}_init ----") - val bulkEdges = (for ((ts, op, props) <- opWithProps) yield { + val bulkEdges = for ((ts, op, props) <- opWithProps) yield { TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props) - }) + } TestUtil.insertEdgesSync(bulkEdges: _*) @@ -204,15 +234,15 @@ class CrudTest extends IntegrateCommon { case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId) } - val qId = if (labelName == testLabelName) id else "\"" + id + "\"" + val qId = if (labelName == testLabelNameV3 || labelName == testLabelNameV4) id else "\"" + id + "\"" val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL) val jsResult = TestUtil.getEdgesSync(query) val results = jsResult \ "results" - val deegrees = (jsResult \ "degrees").as[List[JsObject]] + val degrees = (jsResult \ "degrees").as[List[JsObject]] val propsLs = (results \\ "props").seq - (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1) + (degrees.head \ LabelMeta.degree.name).as[Int] should be(1) val from = (results \\ "from").seq.last.toString.replaceAll("\"", "") val to = (results \\ "to").seq.last.toString.replaceAll("\"", "") @@ -242,4 +272,4 @@ class CrudTest extends IntegrateCommon { "cacheTTL": $cacheTTL }]]}""") } } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala index 225d3963..107bb2f5 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,24 +20,21 @@ package org.apache.s2graph.core.Integrate import com.typesafe.config._ +import org.apache.s2graph.core._ import org.apache.s2graph.core.mysqls.Label import org.apache.s2graph.core.rest.{RequestParser, RestHandler} import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.core.{Graph, GraphUtil, Management, PostProcess} import org.scalatest._ import play.api.libs.json.{JsValue, Json} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} +import scala.util.Random -trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { +trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll with TestCommon{ import TestUtil._ - var graph: Graph = _ - var parser: RequestParser = _ - var management: Management = _ - var config: Config = _ override def beforeAll = { config = ConfigFactory.load() @@ -51,6 +48,9 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { graph.shutdown() } + def getLabelName(ver: String, consistency: String = "strong") = s"s2graph_label_test_${ver}_${consistency}" + def getLabelName2(ver: String, consistency: String = "strong") = s"s2graph_label_test_${ver}_${consistency}_2" + /** * Make Service, Label, Vertex for integrate test */ @@ -67,14 +67,16 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) println(s">> Service created : $createService, $tryRes") - val labelNames = Map(testLabelName -> testLabelNameCreate, - testLabelName2 -> testLabelName2Create, - testLabelNameV1 -> testLabelNameV1Create, - testLabelNameWeak -> testLabelNameWeakCreate) + // val labelNames = Map(testLabelNameV4 -> testLabelNameCreate(ver), + // testLabelNameV3 -> testLabelNameCreate(ver), + // testLabelNameV1 -> testLabelNameCreate(ver), + // testLabelNameWeak -> testLabelNameWeakCreate(ver)) - for { - (labelName, create) <- labelNames - } { + // Create test labels by versions + consistency. + for (n <- versions; consistency <- Seq("strong", "weak")) yield { + val ver = s"v$n" + val labelName = getLabelName(ver, consistency) + val create = testLabelCreate(ver, consistency) Management.deleteLabel(labelName) Label.findByName(labelName, useCache = false) match { case None => @@ -117,6 +119,40 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { // s // } + def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { + Json.parse( + s""" + |[ + |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}]} + |] + """.stripMargin) + } + + def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { + ids.map { id => + Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) + } + } + + val vertexPropsKeys = List( + ("age", "int") + ) + + def randomProps() = { + (for { + (propKey, propType) <- vertexPropsKeys + } yield { + propKey -> Random.nextInt(100) + }).toMap + } + + def getVerticesSync(queryJson: JsValue): JsValue = { + val restHandler = new RestHandler(graph) + logger.info(Json.prettyPrint(queryJson)) + val f = restHandler.getVertices(queryJson) + Await.result(f, HttpRequestWaitingTime) + } + def deleteAllSync(jsValue: JsValue) = { val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json => val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json) @@ -128,12 +164,33 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { Await.result(future, HttpRequestWaitingTime) } + def getEdgesSync(queryJson: JsValue): JsValue = { logger.info(Json.prettyPrint(queryJson)) val restHandler = new RestHandler(graph) Await.result(restHandler.getEdgesAsync(queryJson)(PostProcess.toSimpleVertexArrJson), HttpRequestWaitingTime) } + def checkEdgesSync(checkEdgeJson: JsValue): JsValue = { + logger.info(Json.prettyPrint(checkEdgeJson)) + + val ret = parser.toCheckEdgeParam(checkEdgeJson) match { + case (e, _) => graph.checkEdges(e) + } + val result = Await.result(ret, HttpRequestWaitingTime) + val jsResult = PostProcess.toSimpleVertexArrJson(result) + + logger.info(jsResult.toString) + jsResult + } + + def mutateEdgesSync(bulkEdges: String*) = { + val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true) + val jsResult = Await.result(req, HttpRequestWaitingTime) + + jsResult + } + def insertEdgesSync(bulkEdges: String*) = { val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true) val jsResult = Await.result(req, HttpRequestWaitingTime) @@ -150,10 +207,14 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { // common tables val testServiceName = "s2graph" - val testLabelName = "s2graph_label_test" - val testLabelName2 = "s2graph_label_test_2" - val testLabelNameV1 = "s2graph_label_test_v1" - val testLabelNameWeak = "s2graph_label_test_weak" + val testLabelNameV4 = "s2graph_label_test_v4_strong" + val testLabelNameV3 = "s2graph_label_test_v3_strong" + val testLabelNameV2 = "s2graph_label_test_v2_strong" + val testLabelNameV1 = "s2graph_label_test_v1_strong" + val testLabelNameWeakV4 = "s2graph_label_test_v4_weak" + val testLabelNameWeakV3 = "s2graph_label_test_v3_weak" + val testLabelNameWeakV2 = "s2graph_label_test_v2_weak" + val testLabelNameWeakV1 = "s2graph_label_test_v1_weak" val testColumnName = "user_id_test" val testColumnType = "long" val testTgtColumnName = "item_id_test" @@ -167,14 +228,15 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { val createService = s"""{"serviceName" : "$testServiceName"}""" - val testLabelNameCreate = + def testLabelCreate(ver: String, consistency: String = "strong") = { + val label = getLabelName(ver, consistency) s""" { - "label": "$testLabelName", + "label": "$label", "srcServiceName": "$testServiceName", "srcColumnName": "$testColumnName", "srcColumnType": "long", - "tgtServiceName": "$testServiceName", + "tgtServiceName": "$testServiceName", "tgtColumnName": "$testColumnName", "tgtColumnType": "long", "indices": [ @@ -203,23 +265,27 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { "defaultValue": false } ], - "consistencyLevel": "strong", - "schemaVersion": "v4", - "compressionAlgorithm": "gz", - "hTableName": "$testHTableName" + "consistencyLevel": "$consistency", + "schemaVersion": "$ver", + "compressionAlgorithm": "gz" }""" + } - val testLabelName2Create = + def testLabel2Create(ver: String, consistency: String = "strong") = { + val label = getLabelName2(ver, consistency) s""" { - "label": "$testLabelName2", + "label": "$label", "srcServiceName": "$testServiceName", "srcColumnName": "$testColumnName", "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], + "tgtServiceName": "$testServiceName", + "tgtColumnName": "$testColumnName", + "tgtColumnType": "long", + "indices": [ + {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, + {"name": "$index2", "propNames": ["_timestamp"]} + ], "props": [ { "name": "time", @@ -242,87 +308,295 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll { "defaultValue": false } ], - "consistencyLevel": "strong", - "isDirected": false, - "schemaVersion": "v3", + "consistencyLevel": "$consistency", + "schemaVersion": "$ver", "compressionAlgorithm": "gz" }""" + } - val testLabelNameV1Create = + def querySingle(id: Int, label: String, offset: Int = 0, limit: Int = 100) = Json.parse( s""" - { - "label": "$testLabelNameV1", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "${testTgtColumnName}_v1", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$label", + "direction": "out", + "offset": $offset, + "limit": $limit + } + ]] + } + """) + + def queryWithInterval(label: String, id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$label", + "index": "$index", + "interval": { + "from": [ { "$prop": $fromVal } ], + "to": [ { "$prop": $toVal } ] + } + } + ]] + } + """) + + def queryWhere(id: Int, label: String, where: String) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "limit": 100, + "where": "${where}" + } + ]] + }""") + + def queryExclude(id: Int, label: String) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "limit": 2 + }, + { + "label": "${label}", + "direction": "in", + "offset": 0, + "limit": 2, + "exclude": true + } + ]] + }""") + + def queryGroupBy(id: Int, label: String, props: Seq[String]): JsValue = { + Json.obj( + "groupBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label + ) + ) + ) + ) + ) } - ], - "consistencyLevel": "strong", - "isDirected": true, - "schemaVersion": "v1", - "compressionAlgorithm": "gz" - }""" - val testLabelNameWeakCreate = + def queryTransform(id: Int, label: String, transforms: String) = Json.parse( s""" - { - "label": "$testLabelNameWeak", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": 0, + "transform": $transforms + } + ]] + }""") + + def queryIndex(ids: Seq[Int], label: String, indexName: String) = { + val $from = Json.arr( + Json.obj("serviceName" -> testServiceName, + "columnName" -> testColumnName, + "ids" -> ids)) + + val $step = Json.arr(Json.obj("label" -> label, "index" -> indexName)) + val $steps = Json.arr(Json.obj("step" -> $step)) + + val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps) + js } - ], - "consistencyLevel": "weak", - "isDirected": true, - "compressionAlgorithm": "gz" - }""" + + def queryParents(id: Long) = Json.parse( + s""" + { + "returnTree": true, + "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + [ { + "label": "$testLabelNameV3", + "direction": "out", + "offset": 0, + "limit": 2 + } + ],[{ + "label": "$testLabelNameV3", + "direction": "in", + "offset": 0, + "limit": -1 + } + ]] + }""".stripMargin) + + def querySingleWithTo(id: Int, label: String, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${label}", + "direction": "out", + "offset": $offset, + "limit": $limit, + "_to": $to + } + ]] + } + """) + + def queryScore(id: Int, label: String, scoring: Map[String, Int]): JsValue = Json.obj( + "srcVertices" -> Json.arr( + Json.obj( + "serviceName" -> testServiceName, + "columnName" -> testColumnName, + "id" -> id + ) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label, + "scoring" -> scoring + ) + ) + ) + ) + ) + + def queryOrderBy(id: Int, label: String, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj( + "orderBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> label, + "scoring" -> scoring + ) + ) + ) + ) + ) + + def queryWithSampling(id: Int, label: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + } + ] + }""") + + def twoStepQueryWithSampling(id: Int, label: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + }, + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": $sample + }] + } + ] + }""") + + def twoQueryWithSampling(id: Int, label: String, label2: String, sample: Int) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "id": $id + }], + "steps": [ + { + "step": [{ + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 50, + "sample": $sample + }, + { + "label": "$label", + "direction": "out", + "offset": 0, + "limit": 50 + }] + } + ] + }""") } -} + +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala index 8bdc99b4..de409746 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala @@ -1,27 +1,8 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.s2graph.core.Integrate import org.apache.s2graph.core.utils.logger import org.scalatest.BeforeAndAfterEach -import play.api.libs.json.{JsNumber, JsValue, Json} +import play.api.libs.json._ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { @@ -32,1064 +13,232 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { val weight = "weight" val is_hidden = "is_hidden" - test("interval") { - def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "index": "$index", - "interval": { - "from": [ { "$prop": $fromVal } ], - "to": [ { "$prop": $toVal } ] - } - } - ]] - } - """) - - var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index - (edges \ "size").toString should be("1") + versions map { n => - edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index - (edges \ "size").toString should be("2") + val ver = s"v$n" + val label = getLabelName(ver) + val label2 = getLabelName2(ver) + val tag = getTag(ver) - edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index - (edges \ "size").toString should be("1") + test(s"interval $ver", tag) { - edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index - (edges \ "size").toString should be("2") - } + var edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index + (edges \ "size").toString should be("1") - test("get edge with where condition") { - def queryWhere(id: Int, where: String) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "where": "${where}" - } - ]] - }""") + edges = getEdgesSync(queryWithInterval(label, 0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index + (edges \ "size").toString should be("2") - var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)")) - (result \ "results").as[List[JsValue]].size should be(1) + edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 11)) // test interval on weight index + (edges \ "size").toString should be("1") - result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)")) - (result \ "results").as[List[JsValue]].size should be(1) + edges = getEdgesSync(queryWithInterval(label, 2, index1, "weight", 10, 20)) // test interval on weight index + (edges \ "size").toString should be("2") + } - result = getEdgesSync(queryWhere(0, "_from=0")) - (result \ "results").as[List[JsValue]].size should be(2) + test(s"get edge with where condition $ver", tag) { - result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)")) - (result \ "results").as[List[JsValue]].size should be(2) + var result = getEdgesSync(queryWhere(0, label, "is_hidden=false and _from in (-1, 0)")) + (result \ "results").as[List[JsValue]].size should be(1) - result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)")) - (result \ "results").as[List[JsValue]].size should be(2) - } + result = getEdgesSync(queryWhere(0, label, "is_hidden=true and _to in (1)")) + (result \ "results").as[List[JsValue]].size should be(1) - test("get edge exclude") { - def queryExclude(id: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 2 - }, - { - "label": "${testLabelName}", - "direction": "in", - "offset": 0, - "limit": 2, - "exclude": true - } - ]] - }""") + result = getEdgesSync(queryWhere(0, label, "_from=0")) + (result \ "results").as[List[JsValue]].size should be(2) - val result = getEdgesSync(queryExclude(0)) - (result \ "results").as[List[JsValue]].size should be(1) - } + result = getEdgesSync(queryWhere(2, label, "_from=2 or weight in (-1)")) + (result \ "results").as[List[JsValue]].size should be(2) - test("get edge groupBy property") { - def queryGroupBy(id: Int, props: Seq[String]): JsValue = { - Json.obj( - "groupBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName - ) - ) - ) - ) - ) + result = getEdgesSync(queryWhere(2, label, "_from=2 and weight in (10, 20)")) + (result \ "results").as[List[JsValue]].size should be(2) } - val result = getEdgesSync(queryGroupBy(0, Seq("weight"))) - (result \ "size").as[Int] should be(2) - val weights = (result \ "results" \\ "groupBy").map { js => - (js \ "weight").as[Int] + test(s"get edge exclude $ver", tag) { + val result = getEdgesSync(queryExclude(0, label)) + (result \ "results").as[List[JsValue]].size should be(1) } - weights should contain(30) - weights should contain(40) - - weights should not contain (10) - } - test("edge transform") { - def queryTransform(id: Int, transforms: String) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "transform": $transforms - } - ]] - }""") + test(s"get edge groupBy property $ver", tag) { - var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]")) - (result \ "results").as[List[JsValue]].size should be(2) + val result = getEdgesSync(queryGroupBy(0, label, Seq("weight"))) + (result \ "size").as[Int] should be(2) + val weights = (result \\ "groupBy").map { js => + (js \ "weight").as[Int] + } + weights should contain(30) + weights should contain(40) - result = getEdgesSync(queryTransform(0, "[[\"weight\"]]")) - (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "weight").map(_.toString).sorted) - - result = getEdgesSync(queryTransform(0, "[[\"_from\"]]")) - (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "from").map(_.toString).sorted) - } - - test("index") { - def queryIndex(ids: Seq[Int], indexName: String) = { - val $from = Json.arr( - Json.obj("serviceName" -> testServiceName, - "columnName" -> testColumnName, - "ids" -> ids)) - - val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName)) - val $steps = Json.arr(Json.obj("step" -> $step)) - - val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps) - js + weights should not contain (10) } - // weight order - var result = getEdgesSync(queryIndex(Seq(0), "idx_1")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40)) - - // timestamp order - result = getEdgesSync(queryIndex(Seq(0), "idx_2")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30)) - } - - // "checkEdges" in { - // running(FakeApplication()) { - // val json = Json.parse( s""" - // [{"from": 0, "to": 1, "label": "$testLabelName"}, - // {"from": 0, "to": 2, "label": "$testLabelName"}] - // """) - // - // def checkEdges(queryJson: JsValue): JsValue = { - // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get - // contentAsJson(ret) - // } - // - // val res = checkEdges(json) - // val typeRes = res.isInstanceOf[JsArray] - // typeRes must equalTo(true) - // - // val fst = res.as[Seq[JsValue]].head \ "to" - // fst.as[Int] must equalTo(1) - // - // val snd = res.as[Seq[JsValue]].last \ "to" - // snd.as[Int] must equalTo(2) - // } - // } - - + test(s"edge transform $ver", tag) { - test("duration") { - def queryDuration(ids: Seq[Int], from: Int, to: Int) = { - val $from = Json.arr( - Json.obj("serviceName" -> testServiceName, - "columnName" -> testColumnName, - "ids" -> ids)) + var result = getEdgesSync(queryTransform(0, label, "[[\"_to\"]]")) + (result \ "results").as[List[JsValue]].size should be(2) - val $step = Json.arr(Json.obj( - "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100, - "duration" -> Json.obj("from" -> from, "to" -> to))) + result = getEdgesSync(queryTransform(0, label, "[[\"weight\"]]")) + (result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted) - val $steps = Json.arr(Json.obj("step" -> $step)) - - Json.obj("srcVertices" -> $from, "steps" -> $steps) + result = getEdgesSync(queryTransform(0, label, "[[\"_from\"]]")) + val results = (result \ "results").as[JsValue] + (result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted) } - // get all - var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size should be(4) - // inclusive, exclusive - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size should be(3) - - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size should be(1) - val bulkEdges = Seq( - toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), - toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), - toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)), - toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)) - ) - insertEdgesSync(bulkEdges: _*) + test(s"index $ver", tag) { + // weight order + var result = getEdgesSync(queryIndex(Seq(0), label, "idx_1")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40)) - // duration test after udpate - // get all - result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size should be(4) - - // inclusive, exclusive - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size should be(3) - - result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size should be(1) - - } + // timestamp order + result = getEdgesSync(queryIndex(Seq(0), label, "idx_2")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30)) + } - test("return tree") { - def queryParents(id: Long) = Json.parse( - s""" - { - "returnTree": true, - "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 2 - } - ],[{ - "label": "$testLabelName", - "direction": "in", - "offset": 0, - "limit": -1 - } - ]] - }""".stripMargin) + test(s"return tree $ver", tag) { + val src = 100 + val tgt = 200 - val src = 100 - val tgt = 200 + insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, label)) - insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName)) + val result = TestUtil.getEdgesSync(queryParents(src)) + val parents = (result \ "results").as[Seq[JsValue]] + val ret = parents.forall { + edge => (edge \ "parents").as[Seq[JsValue]].size == 1 + } - val result = TestUtil.getEdgesSync(queryParents(src)) - val parents = (result \ "results").as[Seq[JsValue]] - val ret = parents.forall { - edge => (edge \ "parents").as[Seq[JsValue]].size == 1 + ret should be(true) } - ret should be(true) - } - -// test("pagination and _to") { -// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( -// s""" -// { "srcVertices": [ -// { "serviceName": "${testServiceName}", -// "columnName": "${testColumnName}", -// "id": ${id} -// }], -// "steps": [ -// [ { -// "label": "${testLabelName}", -// "direction": "out", -// "offset": $offset, -// "limit": $limit, -// "_to": $to -// } -// ]] -// } -// """) -// -// val src = System.currentTimeMillis().toInt -// -// val bulkEdges = Seq( -// toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), -// toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), -// toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)), -// toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40)) -// ) -// insertEdgesSync(bulkEdges: _*) -// -// var result = getEdgesSync(querySingle(src, offset = 0, limit = 2)) -// var edges = (result \ "results").as[List[JsValue]] -// -// edges.size should be(2) -// (edges(0) \ "to").as[Long] should be(4) -// (edges(1) \ "to").as[Long] should be(3) -// -// result = getEdgesSync(querySingle(src, offset = 1, limit = 2)) -// -// edges = (result \ "results").as[List[JsValue]] -// edges.size should be(2) -// (edges(0) \ "to").as[Long] should be(3) -// (edges(1) \ "to").as[Long] should be(2) -// -// result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1)) -// edges = (result \ "results").as[List[JsValue]] -// edges.size should be(1) -// } - - test("order by") { - def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj( - "srcVertices" -> Json.arr( - Json.obj( - "serviceName" -> testServiceName, - "columnName" -> testColumnName, - "id" -> id - ) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) - ) - ) - def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj( - "orderBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) + // test(s"pagination and _to $ver", tag) { + // val src = System.currentTimeMillis().toInt + // + // val bulkEdges = Seq( + // toEdge(1001, insert, e, src, 1, label, Json.obj(weight -> 10, is_hidden -> true)), + // toEdge(2002, insert, e, src, 2, label, Json.obj(weight -> 20, is_hidden -> false)), + // toEdge(3003, insert, e, src, 3, label, Json.obj(weight -> 30)), + // toEdge(4004, insert, e, src, 4, label, Json.obj(weight -> 40)) + // ) + // insertEdgesSync(bulkEdges: _*) + // + // var result = getEdgesSync(querySingle(src, label, offset = 1, limit = 2)) + // + // var edges = (result \ "results").as[List[JsValue]] + // + // edges.size should be(2) + // (edges(0) \ "to").as[Long] should be(4) + // (edges(1) \ "to").as[Long] should be(3) + // + // result = getEdgesSync(querySingle(src, label, offset = 1, limit = 2)) + // + // edges = (result \ "results").as[List[JsValue]] + // edges.size should be(2) + // (edges(0) \ "to").as[Long] should be(3) + // (edges(1) \ "to").as[Long] should be(2) + // + // result = getEdgesSync(querySingleWithTo(src, label, offset = 0, limit = -1, to = 1)) + // edges = (result \ "results").as[List[JsValue]] + // edges.size should be(1) + // } + + test(s"order by $ver", tag) { + val bulkEdges = Seq( + toEdge(1001, insert, e, 0, 1, label, Json.obj(weight -> 10, is_hidden -> true)), + toEdge(2002, insert, e, 0, 2, label, Json.obj(weight -> 20, is_hidden -> false)), + toEdge(3003, insert, e, 2, 0, label, Json.obj(weight -> 30)), + toEdge(4004, insert, e, 2, 1, label, Json.obj(weight -> 40)) ) - ) - - val bulkEdges = Seq( - toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)), - toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)), - toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)), - toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)) - ) - - insertEdgesSync(bulkEdges: _*) - - // get edges - val edges = getEdgesSync(queryScore(0, Map("weight" -> 1))) - val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) - val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) - - val edgesTo = edges \ "results" \\ "to" - val orderByTo = orderByScore \ "results" \\ "to" - val ascOrderByTo = ascOrderByScore \ "results" \\ "to" - - edgesTo should be(Seq(JsNumber(2), JsNumber(1))) - edgesTo should be(orderByTo) - ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2))) - edgesTo.reverse should be(ascOrderByTo) - } - - - test("query with sampling") { - def queryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - } - ] - }""") - - def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - }, - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": $sample - }] - } - ] - }""") - - def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - { - "step": [{ - "label": "$testLabelName", - "direction": "out", - "offset": 0, - "limit": 50, - "sample": $sample - }, - { - "label": "$testLabelName2", - "direction": "out", - "offset": 0, - "limit": 50 - }] - } - ] - }""") - - val sampleSize = 2 - val ts = "1442985659166" - val testId = 22 - - val bulkEdges = Seq( - toEdge(ts, insert, e, testId, 122, testLabelName), - toEdge(ts, insert, e, testId, 222, testLabelName), - toEdge(ts, insert, e, testId, 322, testLabelName), - toEdge(ts, insert, e, testId, 922, testLabelName2), - toEdge(ts, insert, e, testId, 222, testLabelName2), - toEdge(ts, insert, e, testId, 322, testLabelName2), + insertEdgesSync(bulkEdges: _*) - toEdge(ts, insert, e, 122, 1122, testLabelName), - toEdge(ts, insert, e, 122, 1222, testLabelName), - toEdge(ts, insert, e, 122, 1322, testLabelName), - toEdge(ts, insert, e, 222, 2122, testLabelName), - toEdge(ts, insert, e, 222, 2222, testLabelName), - toEdge(ts, insert, e, 222, 2322, testLabelName), - toEdge(ts, insert, e, 322, 3122, testLabelName), - toEdge(ts, insert, e, 322, 3222, testLabelName), - toEdge(ts, insert, e, 322, 3322, testLabelName) - ) + // get edges + val edges = getEdgesSync(queryScore(0, label, Map("weight" -> 1))) + val orderByScore = getEdgesSync(queryOrderBy(0, label, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) + val ascOrderByScore = getEdgesSync(queryOrderBy(0, label, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) - insertEdgesSync(bulkEdges: _*) + val edgesTo = edges \ "results" \\ "to" + val orderByTo = orderByScore \ "results" \\ "to" + val ascOrderByTo = ascOrderByScore \ "results" \\ "to" - val result1 = getEdgesSync(queryWithSampling(testId, sampleSize)) - (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size)) - - val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize)) - (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) - - val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize)) - (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3 - } - test("test query with filterOut query") { - def queryWithFilterOut(id1: String, id2: String) = Json.parse( - s"""{ - | "limit": 10, - | "filterOut": { - | "srcVertices": [{ - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | }], - | "steps": [{ - | "step": [{ - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | }] - | }] - | }, - | "srcVertices": [{ - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | }], - | "steps": [{ - | "step": [{ - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | }] - | }] - |} - """.stripMargin - ) - - val testId1 = "-23" - val testId2 = "-25" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 111, testLabelName, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 333, testLabelName, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 555, testLabelName, Json.obj(weight -> 1)) - ) - logger.debug(s"${bulkEdges.mkString("\n")}") - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithFilterOut(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - (results(0) \ "to").toString should be("555") - } - - - /** note that this merge two different label result into one */ - test("weighted union") { - def queryWithWeightedUnion(id1: String, id2: String) = Json.parse( - s""" - |{ - | "limit": 10, - | "weights": [ - | 10, - | 1 - | ], - | "groupBy": ["weight"], - | "queries": [ - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | }, - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId1 = "1" - val testId2 = "2" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 444, testLabelName2, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 666, testLabelName2, Json.obj(weight -> 1)) - ) - - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithWeightedUnion(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - (results(0) \ "scoreSum").as[Float] should be(30.0) - (results(0) \ "agg").as[List[JsValue]].size should be(3) - (results(1) \ "scoreSum").as[Float] should be(3.0) - (results(1) \ "agg").as[List[JsValue]].size should be(3) - } - - test("weighted union with options") { - def queryWithWeightedUnionWithOptions(id1: String, id2: String) = Json.parse( - s""" - |{ - | "limit": 10, - | "weights": [ - | 10, - | 1 - | ], - | "groupBy": ["to"], - | "select": ["to", "weight"], - | "filterOut": { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | } - | ] - | }, - | "queries": [ - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id1 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | }, - | { - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id2 - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 5 - | } - | ] - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId1 = "-192848" - val testId2 = "-193849" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)), - toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)), - toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)), - toEdge(4, insert, e, testId2, 111, testLabelName2, Json.obj(weight -> 1)), - toEdge(5, insert, e, testId2, 333, testLabelName2, Json.obj(weight -> 1)), - toEdge(6, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)) - ) - - insertEdgesSync(bulkEdges: _*) - - val rs = getEdgesSync(queryWithWeightedUnionWithOptions(testId1, testId2)) - logger.debug(Json.prettyPrint(rs)) - val results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - - } - - test("scoreThreshold") { - def queryWithScoreThreshold(id: String, scoreThreshold: Int) = Json.parse( - s"""{ - | "limit": 10, - | "scoreThreshold": $scoreThreshold, - | "groupBy": ["to"], - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | }, - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10 - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - val testId = "-23903" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 102, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 101, 104, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 102, 103, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 102, 104, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, 103, 105, testLabelName, Json.obj(weight -> 10)) - ) - // expected: 104 -> 2, 103 -> 2, 102 -> 1,, 105 -> 1 - insertEdgesSync(bulkEdges: _*) - - var rs = getEdgesSync(queryWithScoreThreshold(testId, 2)) - logger.debug(Json.prettyPrint(rs)) - var results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - - rs = getEdgesSync(queryWithScoreThreshold(testId, 1)) - logger.debug(Json.prettyPrint(rs)) - - results = (rs \ "results").as[List[JsValue]] - results.size should be(4) - } - - test("scorePropagateOp test") { - def queryWithPropertyOp(id: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "index": "idx_1", - | "scoring": { - | "weight":1, - | "time": 0 - | }, - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - def querySingleVertexWithOp(id: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "id": $id - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "countSum", - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - - def queryMultiVerticesWithOp(id: String, id2: String, op: String, shrinkageVal: Long) = Json.parse( - s"""{ - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "sum", - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "ids": [$id, $id2] - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "label": "$testLabelName", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "groupBy": ["from"], - | "duplicate": "countSum", - | "transform": [["_from"]] - | } - | ] - | }, { - | "step": [ - | { - | "label": "$testLabelName2", - | "direction": "out", - | "offset": 0, - | "limit": 10, - | "scorePropagateOp": "$op", - | "scorePropagateShrinkage": $shrinkageVal - | } - | ] - | } - | ] - |} - """.stripMargin - ) - val testId = "-30000" - val testId2 = "-4000" - - val bulkEdges = Seq( - toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId, 102, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 103, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 104, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId, 105, testLabelName2, Json.obj(weight -> 10)), + edgesTo should be(Seq(JsNumber(2), JsNumber(1))) + edgesTo should be(orderByTo) + ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2))) + edgesTo.reverse should be(ascOrderByTo) + } - toEdge(1, insert, e, testId2, 101, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 102, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 103, testLabelName, Json.obj(weight -> -10)), - toEdge(1, insert, e, testId2, 102, testLabelName2, Json.obj(weight -> 10)), - toEdge(1, insert, e, testId2, 105, testLabelName2, Json.obj(weight -> 10)) - ) - insertEdgesSync(bulkEdges: _*) + test(s"query with sampling $ver", tag) { + + val sampleSize = 2 + val ts = "1442985659166" + val testId = 22 + + val bulkEdges = Seq( + toEdge(ts, insert, e, testId, 122, label), + toEdge(ts, insert, e, testId, 222, label), + toEdge(ts, insert, e, testId, 322, label), + + toEdge(ts, insert, e, testId, 922, label2), + toEdge(ts, insert, e, testId, 222, label2), + toEdge(ts, insert, e, testId, 322, label2), + + toEdge(ts, insert, e, 122, 1122, label), + toEdge(ts, insert, e, 122, 1222, label), + toEdge(ts, insert, e, 122, 1322, label), + toEdge(ts, insert, e, 222, 2122, label), + toEdge(ts, insert, e, 222, 2222, label), + toEdge(ts, insert, e, 222, 2322, label), + toEdge(ts, insert, e, 322, 3122, label), + toEdge(ts, insert, e, 322, 3222, label), + toEdge(ts, insert, e, 322, 3322, label) + ) - val firstStepEdgeCount = 3l - val secondStepEdgeCount = 4l + insertEdgesSync(bulkEdges: _*) + val result0 = getEdgesSync(querySingle(testId, label)) + logger.debug(s"2-step sampling res 0: ${Json.prettyPrint(result0)}") + (result0 \ "results").as[List[JsValue]].size should be(3) - var shrinkageVal = 10l - var rs = getEdgesSync(querySingleVertexWithOp(testId, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - var results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - var scoreSum = secondStepEdgeCount.toDouble / (firstStepEdgeCount.toDouble + shrinkageVal) - (results(0) \ "scoreSum").as[Double] should be(scoreSum) + val result1 = getEdgesSync(queryWithSampling(testId, label, sampleSize)) + logger.debug(s"2-step sampling res 1: ${Json.prettyPrint(result1)}") + (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size)) - rs = getEdgesSync(queryMultiVerticesWithOp(testId, testId2, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(2) - scoreSum = secondStepEdgeCount.toDouble / (firstStepEdgeCount.toDouble + shrinkageVal) - (results(0) \ "scoreSum").as[Double] should be(scoreSum) - scoreSum = 2.toDouble / (3.toDouble + shrinkageVal) - (results(1) \ "scoreSum").as[Double] should be(scoreSum) + val result2 = getEdgesSync(twoStepQueryWithSampling(testId, label, sampleSize)) + logger.debug(s"2-step sampling res 2: ${Json.prettyPrint(result2)}") + (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) - // check for divide zero case - shrinkageVal = 30l - rs = getEdgesSync(queryWithPropertyOp(testId, "divide", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - (results(0) \ "scoreSum").as[Double] should be(0) + val result3 = getEdgesSync(twoQueryWithSampling(testId, label, label2, sampleSize)) + logger.debug(s"2-step sampling res 3: ${Json.prettyPrint(result3)}") + (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3 + } - // "plus" operation - rs = getEdgesSync(querySingleVertexWithOp(testId, "plus", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - scoreSum = (firstStepEdgeCount + 1) * secondStepEdgeCount - (results(0) \ "scoreSum").as[Long] should be(scoreSum) - // "multiply" operation - rs = getEdgesSync(querySingleVertexWithOp(testId, "multiply", shrinkageVal)) - logger.debug(Json.prettyPrint(rs)) - results = (rs \ "results").as[List[JsValue]] - results.size should be(1) - scoreSum = (firstStepEdgeCount * 1) * secondStepEdgeCount - (results(0) \ "scoreSum").as[Long] should be(scoreSum) + // "checkEdges" in { + // running(FakeApplication()) { + // val json = Json.parse( s""" + // [{"from": 0, "to": 1, "label": "$testLabelName"}, + // {"from": 0, "to": 2, "label": "$testLabelName"}] + // """) + // + // def checkEdges(queryJson: JsValue): JsValue = { + // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get + // contentAsJson(ret) + // } + // + // val res = checkEdges(json) + // val typeRes = res.isInstanceOf[JsArray] + // typeRes must equalTo(true) + // + // val fst = res.as[Seq[JsValue]].head \ "to" + // fst.as[Int] must equalTo(1) + // + // val snd = res.as[Seq[JsValue]].last \ "to" + // snd.as[Int] must equalTo(2) + // } + // } } - def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "id": $id - }], - "steps": [ - [ { - "label": "$testLabelName", - "direction": "out", - "offset": $offset, - "limit": $limit - } - ]] - } - """) - - def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj( - "limit" -> limit, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName - ) - ) - ) - ) - ) - - // called by each test, each override def beforeEach = initTestData() @@ -1097,25 +246,29 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { override def initTestData(): Unit = { super.initTestData() - insertEdgesSync( - toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)), - toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)), - toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)), - toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)), - toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)), - toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)), - toEdge(1, insert, e, -1, 1000, testLabelName), - toEdge(1, insert, e, -1, 2000, testLabelName), - toEdge(1, insert, e, -1, 3000, testLabelName), - toEdge(1, insert, e, 1000, 10000, testLabelName), - toEdge(1, insert, e, 1000, 11000, testLabelName), - toEdge(1, insert, e, 2000, 11000, testLabelName), - toEdge(1, insert, e, 2000, 12000, testLabelName), - toEdge(1, insert, e, 3000, 12000, testLabelName), - toEdge(1, insert, e, 3000, 13000, testLabelName), - toEdge(1, insert, e, 10000, 100000, testLabelName), - toEdge(2, insert, e, 11000, 200000, testLabelName), - toEdge(3, insert, e, 12000, 300000, testLabelName) - ) + versions map { n => + val ver = s"v$n" + val label = getLabelName(ver) + insertEdgesSync( + toEdge(1000, insert, e, 0, 1, label, Json.obj(weight -> 40, is_hidden -> true)), + toEdge(2000, insert, e, 0, 2, label, Json.obj(weight -> 30, is_hidden -> false)), + toEdge(3000, insert, e, 2, 0, label, Json.obj(weight -> 20)), + toEdge(4000, insert, e, 2, 1, label, Json.obj(weight -> 10)), + toEdge(3000, insert, e, 10, 20, label, Json.obj(weight -> 20)), + toEdge(4000, insert, e, 20, 20, label, Json.obj(weight -> 10)), + toEdge(1, insert, e, -1, 1000, label), + toEdge(1, insert, e, -1, 2000, label), + toEdge(1, insert, e, -1, 3000, label), + toEdge(1, insert, e, 1000, 10000, label), + toEdge(1, insert, e, 1000, 11000, label), + toEdge(1, insert, e, 2000, 11000, label), + toEdge(1, insert, e, 2000, 12000, label), + toEdge(1, insert, e, 3000, 12000, label), + toEdge(1, insert, e, 3000, 13000, label), + toEdge(1, insert, e, 10000, 100000, label), + toEdge(2, insert, e, 11000, 200000, label), + toEdge(3, insert, e, 12000, 300000, label) + ) + } } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala index 99b56f7b..5ef0c4df 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -32,198 +32,205 @@ class StrongLabelDeleteTest extends IntegrateCommon { import StrongDeleteUtil._ import TestUtil._ - test("Strong consistency select") { - insertEdgesSync(bulkEdges(): _*) + override val versions = 2 to 4 // StrongLabelDeleteTest is supported by v2 and above. + versions map { n => - var result = getEdgesSync(query(0)) - (result \ "results").as[List[JsValue]].size should be(2) - result = getEdgesSync(query(10)) - (result \ "results").as[List[JsValue]].size should be(2) - } - - test("Strong consistency deleteAll") { - val deletedAt = 100 - var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) + val ver = s"v$n" + val label = getLabelName(ver) + val tag = getTag(ver) - println(result) - (result \ "results").as[List[JsValue]].size should be(3) + test(s"Strong consistency select $ver", tag) { + insertEdgesSync(bulkEdges(label = label): _*) - val deleteParam = Json.arr( - Json.obj("label" -> testLabelName2, - "direction" -> "in", - "ids" -> Json.arr("20"), - "timestamp" -> deletedAt)) + var result = getEdgesSync(query(0, label)) + (result \ "results").as[List[JsValue]].size should be(2) + result = getEdgesSync(query(10, label)) + (result \ "results").as[List[JsValue]].size should be(2) + } - deleteAllSync(deleteParam) + test(s"Strong consistency deleteAll $ver", tag) { + val deletedAt = 100 + var result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) - result = getEdgesSync(query(11, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + // println(result) + (result \ "results").as[List[JsValue]].size should be(3) - result = getEdgesSync(query(12, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + val deleteParam = Json.arr( + Json.obj("label" -> getLabelName(ver), + "direction" -> "in", + "ids" -> Json.arr("20"), + "timestamp" -> deletedAt)) - result = getEdgesSync(query(10, direction = "out")) - println(result) - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size should be(1) - (result \\ "to").size should be(1) - (result \\ "to").head.as[String] should be("21") + deleteAllSync(deleteParam) - result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(query(11, label, direction = "out")) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) - insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*) + result = getEdgesSync(query(12, label, direction = "out")) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) + result = getEdgesSync(query(10, label, direction = "out")) + // println(result) + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size should be(1) + (result \\ "to").size should be(1) + (result \\ "to").head.as[Long] should be(21l) - (result \ "results").as[List[JsValue]].size should be(3) - } + result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) + // println(result) + (result \ "results").as[List[JsValue]].size should be(0) + insertEdgesSync(bulkEdges(startTs = deletedAt + 1, label): _*) - test("update delete") { - val ret = for { - i <- 0 until testNum - } yield { - val src = (i + 1) * 10000 -// val src = System.currentTimeMillis() + result = getEdgesSync(query(20, label, direction = "in", columnName = testColumnName)) + // println(result) - val (ret, last) = testInner(i, src) - ret should be(true) - ret + (result \ "results").as[List[JsValue]].size should be(3) } - ret.forall(identity) - } - test("update delete 2") { - val src = System.currentTimeMillis() - var ts = 0L + test(s"update delete $ver", tag) { + val ret = for { + i <- 0 until testNum + } yield { + val src = System.currentTimeMillis() - val ret = for { - i <- 0 until testNum - } yield { - val (ret, lastTs) = testInner(ts, src) - val deletedAt = lastTs + 1 - val deletedAt2 = lastTs + 2 - ts = deletedAt2 + 1 // nex start ts + val (ret, last) = testInner(i, src, label) + // val (ret, last) = testInnerFail(i, src) + ret should be(true) + ret + } - ret should be(true) + ret.forall(identity) + } - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) + test(s"update delete 2 $ver", tag) { + val src = System.currentTimeMillis() + var ts = 0L - val deleteRet = deleteAllSync(deleteAllRequest) - val deleteRet2 = deleteAllSync(deleteAllRequest2) + val ret = for { + i <- 0 until testNum + } yield { + val (ret, lastTs) = testInner(ts, src, label) + val deletedAt = lastTs + 1 + val deletedAt2 = lastTs + 2 + ts = deletedAt2 + 1 // next start ts - val result = getEdgesSync(query(id = src)) - println(result) + ret should be(true) - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty should be(true) + val deleteAllRequest = Json.arr(Json.obj("label" -> label, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + val deleteAllRequest2 = Json.arr(Json.obj("label" -> label, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) - val degreeAfterDeleteAll = getDegree(result) + val deleteRet = deleteAllSync(deleteAllRequest) + val deleteRet2 = deleteAllSync(deleteAllRequest2) - degreeAfterDeleteAll should be(0) - degreeAfterDeleteAll === (0) - } + val result = getEdgesSync(query(id = src, label)) + println(s">>>>> ${Json.prettyPrint(result)}") - ret.forall(identity) - } + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty should be(true) - /** This test stress out test on degree - * when contention is low but number of adjacent edges are large - * Large set of contention test - */ - test("large degrees") { - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - // val allRequests = insertRequests ++ deleteRequests - val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => - insertEdgesAsync(bulkRequests: _*) + val degreeAfterDeleteAll = getDegree(result) + + degreeAfterDeleteAll should be(0) + degreeAfterDeleteAll === (0) + } + + ret.forall(identity) } - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + /** This test stress out test on degree + * when contention is low but number of adjacent edges are large + * Large set of contention test + */ + test(s"large degrees $ver", tag) { + val labelName = getLabelName(ver) + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + // val allRequests = insertRequests ++ deleteRequests + val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => + insertEdgesAsync(bulkRequests: _*) + } - val expectedDegree = insertRequests.size - deleteRequests.size - val queryJson = query(id = src) - val result = getEdgesSync(queryJson) - val resultSize = (result \ "size").as[Long] - val resultDegree = getDegree(result) + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - // println(result) + val expectedDegree = insertRequests.size - deleteRequests.size + val queryJson = query(id = src, label) + val result = getEdgesSync(queryJson) + val resultSize = (result \ "size").as[Long] + val resultDegree = getDegree(result) - val ret = resultSize == expectedDegree && resultDegree == resultSize - println(s"[MaxSize]: $maxSize") - println(s"[DeleteSize]: $deleteSize") - println(s"[ResultDegree]: $resultDegree") - println(s"[ExpectedDegree]: $expectedDegree") - println(s"[ResultSize]: $resultSize") - ret should be(true) - } + // println(result) - test("deleteAll") { - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => - insertEdgesAsync(bulkRequests: _*) + val ret = resultSize == expectedDegree && resultDegree == resultSize + // println(s"[MaxSize]: $maxSize") + // println(s"[DeleteSize]: $deleteSize") + // println(s"[ResultDegree]: $resultDegree") + // println(s"[ExpectedDegree]: $expectedDegree") + // println(s"[ResultSize]: $resultSize") + ret should be(true) } - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + test(s"deleteAll $ver", tag) { + val labelName = getLabelName(ver) + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests => + insertEdgesAsync(bulkRequests: _*) + } - val deletedAt = System.currentTimeMillis() - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - deleteAllSync(deleteAllRequest) + // val deletedAt = System.currentTimeMillis() + val deletedAt = src + maxSize + 1000 + 1000 + val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - val result = getEdgesSync(query(id = src)) - println(result) - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty should be(true) + deleteAllSync(deleteAllRequest) - val degreeAfterDeleteAll = getDegree(result) - degreeAfterDeleteAll should be(0) + val result = getEdgesSync(query(id = src, label)) + // println(s"!!!!!RESULT: ${Json.prettyPrint(result)}") + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty should be(true) + + val degreeAfterDeleteAll = getDegree(result) + degreeAfterDeleteAll should be(0) + } } object StrongDeleteUtil { - val labelName = testLabelName2 -// val labelName = testLabelName val maxTgtId = 10 val batchSize = 10 - val testNum = 100 + val testNum = 3 val numOfBatch = 10 - def testInner(startTs: Long, src: Long) = { + def testInner(startTs: Long, src: Long, label: String) = { val lastOps = Array.fill(maxTgtId)("none") var currentTs = startTs @@ -231,14 +238,14 @@ class StrongLabelDeleteTest extends IntegrateCommon { ith <- 0 until numOfBatch jth <- 0 until batchSize } yield { - currentTs += 1 + currentTs += 1 - val tgt = Random.nextInt(maxTgtId) - val op = if (Random.nextDouble() < 0.5) "delete" else "update" + val tgt = Random.nextInt(maxTgtId) + val op = if (Random.nextDouble() < 0.5) "delete" else "update" - lastOps(tgt) = op - Seq(currentTs, op, "e", src, tgt, labelName, "{}").mkString("\t") - } + lastOps(tgt) = op + Seq(currentTs, op, "e", src, tgt, label, "{}").mkString("\t") + } allRequests.foreach(println(_)) @@ -249,7 +256,7 @@ class StrongLabelDeleteTest extends IntegrateCommon { Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) val expectedDegree = lastOps.count(op => op != "delete" && op != "none") - val queryJson = query(id = src) + val queryJson = query(id = src, label = label) val result = getEdgesSync(queryJson) val resultSize = (result \ "size").as[Long] val resultDegree = getDegree(result) @@ -263,19 +270,19 @@ class StrongLabelDeleteTest extends IntegrateCommon { (ret, currentTs) } - def bulkEdges(startTs: Int = 0) = Seq( - toEdge(startTs + 1, "insert", "e", "0", "1", labelName, s"""{"time": 10}"""), - toEdge(startTs + 2, "insert", "e", "0", "1", labelName, s"""{"time": 11}"""), - toEdge(startTs + 3, "insert", "e", "0", "1", labelName, s"""{"time": 12}"""), - toEdge(startTs + 4, "insert", "e", "0", "2", labelName, s"""{"time": 10}"""), - toEdge(startTs + 5, "insert", "e", "10", "20", labelName, s"""{"time": 10}"""), - toEdge(startTs + 6, "insert", "e", "10", "21", labelName, s"""{"time": 11}"""), - toEdge(startTs + 7, "insert", "e", "11", "20", labelName, s"""{"time": 12}"""), - toEdge(startTs + 8, "insert", "e", "12", "20", labelName, s"""{"time": 13}""") + def bulkEdges(startTs: Int = 0, label: String) = Seq( + toEdge(startTs + 1, "insert", "e", "0", "1", label, s"""{"time": 10}"""), + toEdge(startTs + 2, "insert", "e", "0", "1", label, s"""{"time": 11}"""), + toEdge(startTs + 3, "insert", "e", "0", "1", label, s"""{"time": 12}"""), + toEdge(startTs + 4, "insert", "e", "0", "2", label, s"""{"time": 10}"""), + toEdge(startTs + 5, "insert", "e", "10", "20", label, s"""{"time": 10}"""), + toEdge(startTs + 6, "insert", "e", "10", "21", label, s"""{"time": 11}"""), + toEdge(startTs + 7, "insert", "e", "11", "20", label, s"""{"time": 12}"""), + toEdge(startTs + 8, "insert", "e", "12", "20", label, s"""{"time": 13}""") ) - def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName, - _labelName: String = labelName, direction: String = "out") = Json.parse( + def query(id: Long, label: String, serviceName: String = testServiceName, columnName: String = testColumnName, + direction: String = "out") = Json.parse( s""" { "srcVertices": [ { "serviceName": "$serviceName", @@ -284,7 +291,7 @@ class StrongLabelDeleteTest extends IntegrateCommon { }], "steps": [ [ { - "label": "${_labelName}", + "label": "${label}", "direction": "${direction}", "offset": 0, "limit": -1, @@ -299,4 +306,4 @@ class StrongLabelDeleteTest extends IntegrateCommon { } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala similarity index 64% rename from s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala rename to s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala index a1bff685..93f06fc6 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,25 +19,23 @@ package org.apache.s2graph.core.Integrate -import org.apache.s2graph.core.PostProcess +import org.apache.s2graph.core.{CommonTest, PostProcess} import play.api.libs.json.{JsValue, Json} import scala.concurrent.Await -import scala.util.Random -class VertexTestHelper extends IntegrateCommon { +class VertexTest extends IntegrateCommon { import TestUtil._ - import VertexTestHelper._ - test("vertex") { + test("vertex", CommonTest) { val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0) val (serviceName, columnName) = (testServiceName, testColumnName) val data = vertexInsertsPayload(serviceName, columnName, ids) val payload = Json.parse(Json.toJson(data).toString) - println(payload) + println(s">>> insert vertices: ${payload}") val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName)) Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime) @@ -48,6 +46,7 @@ class VertexTestHelper extends IntegrateCommon { val ret = Await.result(res, HttpRequestWaitingTime) val fetched = ret.as[Seq[JsValue]] + println(s">>> fetched vertices: ${fetched}") for { (d, f) <- data.zip(fetched) } yield { @@ -55,36 +54,4 @@ class VertexTestHelper extends IntegrateCommon { ((d \ "props") \ "age") should be((f \ "props") \ "age") } } - - object VertexTestHelper { - def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { - Json.parse( - s""" - |[ - |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")} - ]} - |] - """.stripMargin) - } - - def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { - ids.map { id => - Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) - } - } - - val vertexPropsKeys = List( - ("age", "int") - ) - - def randomProps() = { - (for { - (propKey, propType) <- vertexPropsKeys - } yield { - propKey -> Random.nextInt(100) - }).toMap - } - } -} - - +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala index c0ab3234..82aff438 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -32,30 +32,37 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { import TestUtil._ import WeakLabelDeleteHelper._ - test("test weak consistency select") { - var result = getEdgesSync(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size should be(4) - result = getEdgesSync(query(10)) - println(result) - (result \ "results").as[List[JsValue]].size should be(2) - } - - test("test weak consistency delete") { - var result = getEdgesSync(query(0)) - println(result) - - /** expect 4 edges */ - (result \ "results").as[List[JsValue]].size should be(4) - val edges = (result \ "results").as[List[JsObject]] - val edgesToStore = parser.toEdges(Json.toJson(edges), "delete") - val rets = graph.mutateEdges(edgesToStore, withWait = true) - Await.result(rets, Duration(20, TimeUnit.MINUTES)) - - /** expect noting */ - result = getEdgesSync(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + val label = getLabelName(ver, "weak") + + test(s"test weak consistency select $ver", tag) { + var result = getEdgesSync(queryWeak(0, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(4) + result = getEdgesSync(queryWeak(10, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(2) + } + + test(s"test weak consistency delete $ver", tag) { + var result = getEdgesSync(queryWeak(0, label)) + println(result) + + /** expect 4 edges */ + (result \ "results").as[List[JsValue]].size should be(4) + + val edges = (result \ "results").as[List[JsObject]] + val edgesToStore = parser.toEdges(Json.toJson(edges), "delete") + val rets = graph.mutateEdges(edgesToStore, withWait = true) + Await.result(rets, Duration(20, TimeUnit.MINUTES)) + + /** expect noting */ + result = getEdgesSync(queryWeak(0, label)) + println(result) + (result \ "results").as[List[JsValue]].size should be(0) + } /** insert should be ignored */ /** @@ -63,52 +70,54 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { * This makes sense because hbase think cell is deleted when there are * insert/delete with same timestamp(version) on same cell. * This can be different on different storage system so I think - * this test should be removed. + * this test should be removed. => Indeed, Redis does not support this feature. */ -// val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert") -// val rets2 = graph.mutateEdges(edgesToStore2, withWait = true) -// Await.result(rets2, Duration(20, TimeUnit.MINUTES)) -// -// result = getEdgesSync(query(0)) -// (result \ "results").as[List[JsValue]].size should be(0) - } + // val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert") + // val rets2 = graph.mutateEdges(edgesToStore2, withWait = true) + // Await.result(rets2, Duration(20, TimeUnit.MINUTES)) + // + // result = getEdgesSync(queryWeak(0, label)) + // (result \ "results").as[List[JsValue]].size should be(0) - test("test weak consistency deleteAll") { - val deletedAt = 100 - var result = getEdgesSync(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(3) + test(s"test weak consistency deleteAll $ver", tag) { + val deletedAt = 100 + var result = getEdgesSync(queryWeak(20, label, "in")) + println(result) + (result \ "results").as[List[JsValue]].size should be(3) - val json = Json.arr(Json.obj("label" -> testLabelNameWeak, - "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) - println(json) - deleteAllSync(json) + val json = Json.arr(Json.obj("label" -> label, + "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) + println(json) + deleteAllSync(json) - result = getEdgesSync(query(11, "out")) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(11, label, "out")) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(12, "out")) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(12, label, "out")) + (result \ "results").as[List[JsValue]].size should be(0) - result = getEdgesSync(query(10, "out")) + result = getEdgesSync(queryWeak(10, label, "out")) - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size should be(1) - (result \\ "to").size should be(1) - (result \\ "to").head.as[String] should be("21") + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size should be(1) + (result \\ "to").size should be(1) + (result \\ "to").head.as[Long] should be(21L) - result = getEdgesSync(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size should be(0) + result = getEdgesSync(queryWeak(20, label, "in")) + println(result) + (result \ "results").as[List[JsValue]].size should be(0) - insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*) + insertEdgesSync(bulkEdges(startTs = deletedAt + 1, label): _*) - result = getEdgesSync(query(20, "in", testTgtColumnName)) - (result \ "results").as[List[JsValue]].size should be(3) + result = getEdgesSync(queryWeak(20, label, "in", testColumnName)) + (result \ "results").as[List[JsValue]].size should be(3) + } } + + // called by each test, each override def beforeEach = initTestData() @@ -116,23 +125,26 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { override def initTestData(): Unit = { super.initTestData() - insertEdgesSync(bulkEdges(): _*) + versions map { v => + val ver = s"v$v" + insertEdgesSync(bulkEdges(label = getLabelName(ver, "weak")): _*) + } } object WeakLabelDeleteHelper { - def bulkEdges(startTs: Int = 0) = Seq( - toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""), - toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""), - toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""), - toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""), - toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""), - toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""") + def bulkEdges(startTs: Int = 0, label: String) = Seq( + toEdge(startTs + 1, "insert", "e", "0", "1", label, s"""{"time": 10}"""), + toEdge(startTs + 2, "insert", "e", "0", "1", label, s"""{"time": 11}"""), + toEdge(startTs + 3, "insert", "e", "0", "1", label, s"""{"time": 12}"""), + toEdge(startTs + 4, "insert", "e", "0", "2", label, s"""{"time": 10}"""), + toEdge(startTs + 5, "insert", "e", "10", "20", label, s"""{"time": 10}"""), + toEdge(startTs + 6, "insert", "e", "10", "21", label, s"""{"time": 11}"""), + toEdge(startTs + 7, "insert", "e", "11", "20", label, s"""{"time": 12}"""), + toEdge(startTs + 8, "insert", "e", "12", "20", label, s"""{"time": 13}""") ) - def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse( + def queryWeak(id: Int, label: String, direction: String = "out", columnName: String = testColumnName) = Json.parse( s""" { "srcVertices": [ { "serviceName": "$testServiceName", @@ -141,7 +153,7 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { }], "steps": [ [ { - "label": "${testLabelNameWeak}", + "label": "${label}", "direction": "${direction}", "offset": 0, "limit": 10, @@ -151,5 +163,4 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach { }""") } -} - +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala index 03151882..7e0a0553 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,38 +22,53 @@ package org.apache.s2graph.core import org.apache.s2graph.core.types.{InnerVal, InnerValLike} import org.scalatest.{FunSuite, Matchers} -class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser { +class JsonParserTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels with JSONParser { import InnerVal._ import types.HBaseType._ - val innerValsPerVersion = for { - version <- List(VERSION2, VERSION1) - } yield { - val innerVals = List( - (InnerVal.withStr("ABC123", version), STRING), - (InnerVal.withNumber(23, version), BYTE), - (InnerVal.withNumber(23, version), INT), - (InnerVal.withNumber(Int.MaxValue, version), INT), - (InnerVal.withNumber(Int.MinValue, version), INT), - (InnerVal.withNumber(Long.MaxValue, version), LONG), - (InnerVal.withNumber(Long.MinValue, version), LONG), - (InnerVal.withBoolean(true, version), BOOLEAN) + versions map { n => + + val ver = s"v$n" + val tag = getTag(ver) + + test(s"aa $ver", tag) { + val innerVal = InnerVal.withStr("abc", ver) + val tmp = innerValToJsValue(innerVal, "string") + println(tmp) + } + + test(s"JsValue <-> InnerVal with dataType $ver", tag) { + val (innerValWithDataTypes, version) = innerValsPerVersion(ver) + testInnerValToJsValue(innerValWithDataTypes, version) + } + } + + def innerValsPerVersion(version: String) = { + val innerVals = List( + (InnerVal.withStr("ABC123", version), STRING), + (InnerVal.withNumber(23, version), BYTE), + (InnerVal.withNumber(23, version), INT), + (InnerVal.withNumber(Int.MaxValue, version), INT), + (InnerVal.withNumber(Int.MinValue, version), INT), + (InnerVal.withNumber(Long.MaxValue, version), LONG), + (InnerVal.withNumber(Long.MinValue, version), LONG), + (InnerVal.withBoolean(true, version), BOOLEAN) + ) + val doubleVals = if (version != VERSION1) { + List( + (InnerVal.withDouble(Double.MaxValue, version), DOUBLE), + (InnerVal.withDouble(Double.MinValue, version), DOUBLE), + (InnerVal.withDouble(0.1, version), DOUBLE), + (InnerVal.withFloat(Float.MinValue, version), FLOAT), + (InnerVal.withFloat(Float.MaxValue, version), FLOAT), + (InnerVal.withFloat(0.9f, version), FLOAT) ) - val doubleVals = if (version == VERSION2) { - List( - (InnerVal.withDouble(Double.MaxValue, version), DOUBLE), - (InnerVal.withDouble(Double.MinValue, version), DOUBLE), - (InnerVal.withDouble(0.1, version), DOUBLE), - (InnerVal.withFloat(Float.MinValue, version), FLOAT), - (InnerVal.withFloat(Float.MaxValue, version), FLOAT), - (InnerVal.withFloat(0.9f, version), FLOAT) - ) - } else { - List.empty[(InnerValLike, String)] - } - (innerVals ++ doubleVals, version) + } else { + List.empty[(InnerValLike, String)] } + (innerVals ++ doubleVals, version) + } def testInnerValToJsValue(innerValWithDataTypes: Seq[(InnerValLike, String)], version: String) = { @@ -70,16 +85,5 @@ class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONPar } } - test("aa") { - val innerVal = InnerVal.withStr("abc", VERSION2) - val tmp = innerValToJsValue(innerVal, "string") - println(tmp) - } - test("JsValue <-> InnerVal with dataType") { - for { - (innerValWithDataTypes, version) <- innerValsPerVersion - } { - testInnerValToJsValue(innerValWithDataTypes, version) - } - } -} + +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala index c6de6bba..adc1ff39 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,7 +24,7 @@ import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.JsString class OrderingUtilTest extends FunSuite with Matchers { - test("test SeqMultiOrdering") { + test("test SeqMultiOrdering", CommonTest) { val jsLs: Seq[Seq[Any]] = Seq( Seq(0, "a"), Seq(0, "b"), @@ -48,7 +48,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 1 TupleMultiOrdering") { + test("test tuple 1 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, None, None, None), (0, None, None, None), @@ -71,7 +71,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 2 TupleMultiOrdering") { + test("test tuple 2 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", None, None), (0, "b", None, None), @@ -95,7 +95,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 3 TupleMultiOrdering") { + test("test tuple 3 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", 0l, None), (0, "a", 1l, None), @@ -120,7 +120,7 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } - test("test tuple 4 TupleMultiOrdering") { + test("test tuple 4 TupleMultiOrdering", CommonTest) { val jsLs: Seq[(Any, Any, Any, Any)] = Seq( (0, "a", 0l, JsString("a")), (0, "a", 0l, JsString("b")), @@ -146,4 +146,4 @@ class OrderingUtilTest extends FunSuite with Matchers { resultJsLs.toString() should equal(sortedJsLs.toString()) } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala index 06af38cc..3230aba6 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,20 +24,6 @@ import org.apache.s2graph.core.types.LabelWithDirection import org.scalatest.{FunSuite, Matchers} class QueryParamTest extends FunSuite with Matchers with TestCommon { -// val version = HBaseType.VERSION2 -// val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString) -// test("EdgeTransformer toInnerValOpt") { -// -// /** only labelNameV2 has string type output */ -// val jsVal = Json.arr(Json.arr("_to"), Json.arr("phone_number.$", "phone_number"), Json.arr("age.$", "age")) -// val transformer = EdgeTransformer(queryParamV2, jsVal) -// val convertedLs = transformer.transform(testEdge, None) -// -// convertedLs(0).tgtVertex.innerId.toString == "10" shouldBe true -// convertedLs(1).tgtVertex.innerId.toString == "phone_number.xxxx" shouldBe true -// convertedLs(2).tgtVertex.innerId.toString == "age.20" shouldBe true -// true -// } val dummyRequests = { for { @@ -47,7 +33,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { } } - test("QueryParam toCacheKey bytes") { + test("QueryParam toCacheKey bytes", CommonTest) { val startedAt = System.nanoTime() val queryParam = QueryParam(LabelWithDirection(1, 0)) @@ -66,15 +52,15 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { dummyRequests.zip(dummyRequests).foreach { case (x, y) => val xHash = queryParam.toCacheKey(x) val yHash = queryParam.toCacheKey(y) -// println(xHash, yHash) + // println(xHash, yHash) xHash should be(yHash) } val duration = System.nanoTime() - startedAt - println(s">> bytes: $duration") + // println(s">> bytes: $duration") } - test("QueryParam toCacheKey with variable params") { + test("QueryParam toCacheKey with variable params", CommonTest) { val startedAt = System.nanoTime() val queryParam = QueryParam(LabelWithDirection(1, 0)) @@ -86,7 +72,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { queryParam.limit(1, 10) var yHash = queryParam.toCacheKey(y) queryParam.toCacheKey(x) shouldBe yHash -// println(xHash, yHash) + // println(xHash, yHash) xHash should not be yHash queryParam.limit(0, 10) @@ -99,7 +85,7 @@ class QueryParamTest extends FunSuite with Matchers with TestCommon { val duration = System.nanoTime() - startedAt - println(s">> diff: $duration") + // println(s">> diff: $duration") } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala index 0f8f8338..9b2ab2e3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,12 +19,22 @@ package org.apache.s2graph.core +import com.typesafe.config.{ConfigFactory, Config} import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta} +import org.apache.s2graph.core.rest.RequestParser import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLikeWithTs, LabelWithDirection} trait TestCommon { + + var graph: Graph = _ + var config: Config = ConfigFactory.load() + var management: Management = _ + var parser: RequestParser = _ + + val versions = 1 to 1 + val ts = System.currentTimeMillis() val testServiceId = 1 val testColumnId = 1 @@ -43,6 +53,17 @@ trait TestCommon { def lessThanEqual(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) <= 0 + def getTag(ver: String) = { + ver match { + case "v1" => V1Test + case "v2" => V2Test + case "v3" => V3Test + case "v4" => V4Test + case _ => throw new GraphExceptions.UnsupportedVersionException(s"$ver does no support this feature!") + } + } + + /** */ import HBaseType.{VERSION1, VERSION2} private val tsValSmall = InnerVal.withLong(ts, VERSION1) @@ -204,4 +225,4 @@ trait TestCommon { // new KeyValue(put.key(), put.family(), q, ts, v) // } // } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala index c652e807..bcfa57aa 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,7 +19,6 @@ package org.apache.s2graph.core -import com.typesafe.config.{Config, ConfigFactory} import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} import org.apache.s2graph.core.mysqls.{Label, LabelIndex, Service, ServiceColumn} import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection} @@ -27,27 +26,22 @@ import scalikejdbc.AutoSession import scala.concurrent.ExecutionContext -trait TestCommonWithModels { +trait TestCommonWithModels extends TestCommon { import InnerVal._ - import types.HBaseType._ - var graph: Graph = _ - var config: Config = _ - var management: Management = _ - - def initTests() = { - config = ConfigFactory.load() + def initTests(ver: String) = { + // config = ConfigFactory.load() graph = new Graph(config)(ExecutionContext.Implicits.global) management = new Management(graph) implicit val session = AutoSession - deleteTestLabel() - deleteTestService() + deleteTestLabel(ver) + deleteTestService(ver) - createTestService() - createTestLabel() + createTestService(ver) + createTestLabel(ver) } def zkQuorum = config.getString("hbase.zookeeper.quorum") @@ -56,42 +50,19 @@ trait TestCommonWithModels { implicit val session = AutoSession - val serviceName = "_test_service" - val serviceNameV2 = "_test_service_v2" - val serviceNameV3 = "_test_service_v3" - val serviceNameV4 = "_test_service_v4" + def serviceName(ver: String) = s"_test_service_$ver" + def labelName(ver: String) = s"_test_label_$ver" + def undirectedLabelName(ver: String) = s"_test_label_undirected_$ver" - val columnName = "user_id" - val columnNameV2 = "user_id_v2" - val columnNameV3 = "user_id_v3" - val columnNameV4 = "user_id_v4" + def columnName(ver: String) = s"user_id_$ver" + def tgtColumnName(ver: String) = s"itme_id_$ver" val columnType = "long" - val columnTypeV2 = "long" - val columnTypeV3 = "long" - val columnTypeV4 = "long" - - val tgtColumnName = "itme_id" - val tgtColumnNameV2 = "item_id_v2" - val tgtColumnNameV3 = "item_id_v3" - val tgtColumnNameV4 = "item_id_v4" - val tgtColumnType = "string" - val tgtColumnTypeV2 = "string" - val tgtColumnTypeV3 = "string" - val tgtColumnTypeV4 = "string" val hTableName = "_test_cases" val preSplitSize = 0 - val labelName = "_test_label" - val labelNameV2 = "_test_label_v2" - val labelNameV3 = "_test_label_v3" - val labelNameV4 = "_test_label_v4" - - val undirectedLabelName = "_test_label_undirected" - val undirectedLabelNameV2 = "_test_label_undirected_v2" - val testProps = Seq( Prop("affinity_score", "0.0", DOUBLE), Prop("is_blocked", "false", BOOLEAN), @@ -107,107 +78,43 @@ trait TestCommonWithModels { val hTableTTL = None - def createTestService() = { + def createTestService(ver: String) = { implicit val session = AutoSession - management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV3, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") - management.createService(serviceNameV4, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") + val service = serviceName(ver) + management.createService(service, cluster, hTableName, preSplitSize, hTableTTL = None, "gz") } - def deleteTestService() = { + def deleteTestService(ver: String) = { implicit val session = AutoSession - Management.deleteService(serviceName) - Management.deleteService(serviceNameV2) - Management.deleteService(serviceNameV3) - Management.deleteService(serviceNameV4) + val service = serviceName(ver) + Management.deleteService(service) } - def deleteTestLabel() = { + def deleteTestLabel(ver: String) = { implicit val session = AutoSession - Management.deleteLabel(labelName) - Management.deleteLabel(labelNameV2) - Management.deleteLabel(undirectedLabelName) - Management.deleteLabel(undirectedLabelNameV2) + val label = labelName(ver) + val undirectedLabel = undirectedLabelName(ver) + Management.deleteLabel(label) + Management.deleteLabel(undirectedLabel) } - def createTestLabel() = { + def createTestLabel(ver: String) = { implicit val session = AutoSession - management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType, - isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4") - - management.createLabel(labelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2, - isDirected = true, serviceNameV2, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4") - - management.createLabel(labelNameV3, serviceNameV3, columnNameV3, columnTypeV3, serviceNameV3, tgtColumnNameV3, tgtColumnTypeV3, - isDirected = true, serviceNameV3, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4") - - management.createLabel(labelNameV4, serviceNameV4, columnNameV4, columnTypeV4, serviceNameV4, tgtColumnNameV4, tgtColumnTypeV4, - isDirected = true, serviceNameV4, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4") - - management.createLabel(undirectedLabelName, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType, - isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4") - - management.createLabel(undirectedLabelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2, - isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4") + val label = labelName(ver) + val service = serviceName(ver) + val column = columnName(ver) + management.createLabel(label, service, column, columnType, service, column, columnType, + isDirected = true, service, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, ver, false, "lg4") } - def service = Service.findByName(serviceName, useCache = false).get - - def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get - - def serviceV3 = Service.findByName(serviceNameV3, useCache = false).get - - def serviceV4 = Service.findByName(serviceNameV4, useCache = false).get - - def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get - - def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get - - def columnV3 = ServiceColumn.find(serviceV3.id.get, columnNameV3, useCache = false).get - - def columnV4 = ServiceColumn.find(serviceV4.id.get, columnNameV4, useCache = false).get - - def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get - - def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get - - def tgtColumnV3 = ServiceColumn.find(serviceV3.id.get, tgtColumnNameV3, useCache = false).get - - def tgtColumnV4 = ServiceColumn.find(serviceV4.id.get, tgtColumnNameV4, useCache = false).get - - def label = Label.findByName(labelName, useCache = false).get - - def labelV2 = Label.findByName(labelNameV2, useCache = false).get - - def labelV3 = Label.findByName(labelNameV3, useCache = false).get - - def labelV4 = Label.findByName(labelNameV4, useCache = false).get - - def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get - - def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get - + def service(ver: String) = Service.findByName(serviceName(ver), useCache = false).get + def column(ver: String) = ServiceColumn.find(service(ver).id.get, columnName(ver), useCache = false).get + def tgtColumn(ver: String) = ServiceColumn.find(service(ver).id.get, tgtColumnName(ver), useCache = false).get + def label(ver: String) = Label.findByName(labelName(ver), useCache = false).get + def undirectedLabel(ver: String) = Label.findByName(undirectedLabelName(ver), useCache = false).get def dir = GraphUtil.directions("out") - def op = GraphUtil.operations("insert") - def labelOrderSeq = LabelIndex.DefaultSeq - - def labelWithDir = LabelWithDirection(label.id.get, dir) - - def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir) - - def labelWithDirV3 = LabelWithDirection(labelV3.id.get, dir) - - def labelWithDirV4 = LabelWithDirection(labelV4.id.get, dir) - - def queryParam = QueryParam(labelWithDir) - - def queryParamV2 = QueryParam(labelWithDirV2) - - def queryParamV3 = QueryParam(labelWithDirV3) - - def queryParamV4 = QueryParam(labelWithDirV4) - -} + def labelWithDir(ver: String) = LabelWithDirection(label(ver).id.get, dir) + def queryParam(ver: String) = QueryParam(labelWithDir(ver)) +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala index 66d84e44..34c9b99b 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,131 +19,48 @@ package org.apache.s2graph.core.models -import org.apache.s2graph.core.TestCommonWithModels +import org.apache.s2graph.core.{CommonTest, TestCommonWithModels} import org.apache.s2graph.core.mysqls.Label import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll { override def beforeAll(): Unit = { - initTests() + versions map { v => + val ver = s"v$v" + initTests(ver) + } } override def afterAll(): Unit = { graph.shutdown() } - // val serviceName = "testService" - // val newServiceName = "newTestService" - // val cluster = "localhost" - // val hbaseTableName = "s2graph-dev" - // val columnName = "user_id" - // val columnType = "long" - // val labelName = "model_test_label" - // val newLabelName = "new_model_test_label" - // val columnMetaName = "is_valid_user" - // val labelMetaName = "is_hidden" - // val hbaseTableTTL = -1 - // val id = 1 - // - // val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster, - // "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1)) - // val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get, - // "columnName" -> columnName, "columnType" -> columnType)) - // val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte)) - // val label = HLabel(Map("id" -> id, "label" -> labelName, - // "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType, - // "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType, - // "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get, - // "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1 - // )) - // val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte, - // "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false)) - // val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte, - // "metaSeqs" -> "0", "formular" -> "none")) - test("test Label.findByName") { - val labelOpt = Label.findByName(labelName, useCache = false) - println(labelOpt) - labelOpt.isDefined shouldBe true - val indices = labelOpt.get.indices - indices.size > 0 shouldBe true - println(indices) - val defaultIndexOpt = labelOpt.get.defaultIndex - println(defaultIndexOpt) - defaultIndexOpt.isDefined shouldBe true - val metas = labelOpt.get.metaProps - println(metas) - metas.size > 0 shouldBe true - val srcService = labelOpt.get.srcService - println(srcService) - val tgtService = labelOpt.get.tgtService - println(tgtService) - val service = labelOpt.get.service - println(service) - val srcColumn = labelOpt.get.srcService - println(srcColumn) - val tgtColumn = labelOpt.get.tgtService - println(tgtColumn) + versions map { n => + val ver = s"v$n" + val label = labelName(ver) + test(s"test Label.findByName $ver", CommonTest) { + val labelOpt = Label.findByName(label, useCache = false) + println(labelOpt) + labelOpt.isDefined shouldBe true + val indices = labelOpt.get.indices + indices.size > 0 shouldBe true + println(indices) + val defaultIndexOpt = labelOpt.get.defaultIndex + println(defaultIndexOpt) + defaultIndexOpt.isDefined shouldBe true + val metas = labelOpt.get.metaProps + println(metas) + metas.size > 0 shouldBe true + val srcService = labelOpt.get.srcService + println(srcService) + val tgtService = labelOpt.get.tgtService + println(tgtService) + val service = labelOpt.get.service + println(service) + val srcColumn = labelOpt.get.srcService + println(srcColumn) + val tgtColumn = labelOpt.get.tgtService + println(tgtColumn) + } } - // test("test create") { - // service.create() - // HService.findByName(serviceName, useCache = false) == Some(service) - // - // serviceColumn.create() - // HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn) - // - // columnMeta.create() - // HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta) - // - // label.create() - // HLabel.findByName(labelName, useCache = false) == Some(label) - // - // labelMeta.create() - // HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta) - // - // labelIndex.create() - // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex) - // } - // - // test("test update") { - // service.update("cluster", "...") - // HService.findById(service.id.get, useCache = false).cluster == "..." - // - // service.update("serviceName", newServiceName) - // assert(HService.findByName(serviceName, useCache = false) == None) - // HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get} - // - // label.update("label", newLabelName) - // HLabel.findById(label.id.get, useCache = false).label == "newLabelName" - // - // label.update("consistencyLevel", "strong") - // HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" && - // HLabel.findByName(newLabelName).isDefined && - // HLabel.findByName(labelName) == None - // - // } - // test("test read by index") { - // val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false) - // val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false) - // labels.length == 1 && - // labels.head == label - // idxs.length == 1 && - // idxs.head == labelIndex - // } - // test("test delete") { - //// HLabel.findByName(labelName).foreach { label => - //// label.deleteAll() - //// } - // HLabel.findByName(newLabelName).foreach { label => - // label.deleteAll() - // } - // HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty && - // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty - // - // service.deleteAll() - // } - - // test("test labelIndex") { - // println(HLabelIndex.findByLabelIdAll(1)) - // } - -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala index 4a46e0ed..3c2a9a35 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,6 +22,7 @@ package org.apache.s2graph.core.mysqls import java.util.Properties import com.typesafe.config.ConfigFactory +import org.apache.s2graph.core.CommonTest import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} import scalikejdbc._ @@ -47,7 +48,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } - "Experiment" should "find bucket list" in { + "Experiment" should "find bucket list" taggedAs CommonTest in { Experiment.findBy(1, "exp1") should not be empty Experiment.findBy(1, "exp1").foreach { exp => @@ -56,7 +57,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } } - it should "update bucket list after cache ttl time" in { + it should "update bucket list after cache ttl time" taggedAs CommonTest in { Experiment.findBy(1, "exp1").foreach { exp => val bucket = exp.buckets.head bucket.impressionId should equal("imp1") @@ -80,4 +81,4 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { exp.buckets.head.impressionId should equal("imp2") } } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala index 40166eb7..f990275a 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,65 +21,33 @@ package org.apache.s2graph.core.parsers import org.apache.s2graph.core._ import org.apache.s2graph.core.mysqls.{Label, LabelMeta} -import org.apache.s2graph.core.rest.TemplateHelper import org.apache.s2graph.core.types._ import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.Json -class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() +class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels with TestCommon { - // dummy data for dummy edge - initTests() - - import HBaseType.{VERSION1, VERSION2} + versions map { n => + val ver = s"v$n" + val tag = getTag(ver) + // dummy data for dummy edge + initTests(ver) - val ts = System.currentTimeMillis() - val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion)) + val l = label(ver) + val ld = labelWithDir(ver) - def ids(version: String) = { - val colId = if (version == VERSION2) columnV2.id.get else column.id.get - val srcId = SourceVertexId(colId, InnerVal.withLong(1, version)) - val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version)) - - val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version)) - val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version)) - - val srcVertex = Vertex(srcId, ts) - val tgtVertex = Vertex(tgtId, ts) - val srcVertexStr = Vertex(srcIdStr, ts) - val tgtVertexStr = Vertex(tgtIdStr, ts) - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version) - } - - - def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = { - val whereOpt = WhereParser(label).parse(sql) - whereOpt.isSuccess shouldBe true + val ts = System.currentTimeMillis() + val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, ver)) + val labelMap = Map(l.label -> l) - println("=================================================================") - println(sql) - println(whereOpt.get) + val (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) = ids(ver) - val ret = whereOpt.get.filter(edge) - if (ret != expected) { - println("==================") - println(s"$whereOpt") - println(s"$edge") - println("==================") - } - ret shouldBe expected - } - - test("check where clause not nested") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause not nested $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) - val f = validate(label)(edge) _ + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) + val f = validate(l, labelMap)(edge) _ /** labelName label is long-long relation */ f(s"_to=${tgtVertex.innerId.toString}")(true) @@ -88,18 +56,14 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f(s"_to=19230495")(false) f(s"_to!=19230495")(true) } - } - test("check where clause nested") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause nested $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) - val f = validate(label)(edge) _ + val f = validate(l,labelMap)(edge) _ // time == 3 f("time >= 3")(true) @@ -117,19 +81,13 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f("(time in ( 1,2,4 ) or weight between 1 and 9) or is_hidden= true")(true) f("(time in (1,2,3) or weight between 1 and 10) and is_hidden =false")(false) } - } - test("check where clause with from/to long") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause with from/to long $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir - val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner) - val lname = if (schemaVer == VERSION2) labelNameV2 else labelName - val f = validate(label)(edge) _ + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner) + val f = validate(l, labelMap)(edge) _ f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true) f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true) @@ -137,31 +95,27 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f(s"_from = -1")(false) f(s"_from in (-1, -0.1)")(false) } - } - test("check where clause with parent") { - for { - (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2)) - } { + test(s"check where clause with parent $ver", tag) { /** test for each version */ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 1, "name" -> "abc") val parentJs = Json.obj("is_hidden" -> false, "is_blocked" -> false, "weight" -> 20, "time" -> 3, "name" -> "a") - val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val parentPropsInner = Management.toProps(label, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val propsInner = Management.toProps(l, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs + val parentPropsInner = Management.toProps(l, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs - val grandParentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner) - val parentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner, + val grandParentEdge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, parentPropsInner) + val parentEdge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, parentPropsInner, parentEdges = Seq(EdgeWithScore(grandParentEdge, 1.0))) - val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner, + val edge = Edge(srcVertex, tgtVertex, ld, 0.toByte, ts, propsInner, parentEdges = Seq(EdgeWithScore(parentEdge, 1.0))) println(edge.toString) println(parentEdge.toString) println(grandParentEdge.toString) - val f = validate(label)(edge) _ + val f = validate(l, labelMap)(edge) _ // Compare edge's prop(`_from`) with edge's prop(`name`) f("_from = 1")(true) @@ -184,70 +138,53 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { f("_parent._parent.weight = weight")(false) f("_parent._parent.weight = _parent.weight")(true) } - } - - test("replace reserved") { - val ts = 0 - import TemplateHelper._ - calculate(ts, 1, "hour") should be(hour + ts) - calculate(ts, 1, "day") should be(day + ts) - - calculate(ts + 10, 1, "HOUR") should be(hour + ts + 10) - calculate(ts + 10, 1, "DAY") should be(day + ts + 10) - - val body = """{ - "day": ${1day}, - "hour": ${1hour}, - "-day": "${-10 day}", - "-hour": ${-10 hour}, - "now": "${now}" - } - """ - - val parsed = replaceVariable(ts, body) - val json = Json.parse(parsed) + // test("time decay") { + // val ts = System.currentTimeMillis() + // + // for { + // i <- (0 until 10) + // } { + // val timeUnit = 60 * 60 + // val diff = i * timeUnit + // val x = TimeDecay(1.0, 0.05, timeUnit) + // println(x.decay(diff)) + // } + // } + } - (json \ "day").as[Long] should be (1 * day + ts) - (json \ "hour").as[Long] should be (1 * hour + ts) - (json \ "-day").as[Long] should be (-10 * day + ts) - (json \ "-hour").as[Long] should be (-10 * hour + ts) + def ids(version: String) = { + val colId = column(version).id.get + val srcId = SourceVertexId(colId, InnerVal.withLong(1, version)) + val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version)) - (json \ "now").as[Long] should be (ts) + val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version)) + val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version)) - val otherBody = """{ - "nextday": "${next_day}", - "3dayago": "${next_day - 3 day}", - "nexthour": "${next_hour}" - }""" + val srcVertex = Vertex(srcId, ts) + val tgtVertex = Vertex(tgtId, ts) + val srcVertexStr = Vertex(srcIdStr, ts) + val tgtVertexStr = Vertex(tgtIdStr, ts) + (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version) + } - val currentTs = System.currentTimeMillis() - val expectedDayTs = currentTs / day * day + day - val expectedHourTs = currentTs / hour * hour + hour - val threeDayAgo = expectedDayTs - 3 * day - val currentTsLs = (1 until 1000).map(currentTs + _) + def validate(l: Label, labelMap: Map[String, Label])(edge: Edge)(sql: String)(expected: Boolean) = { + val whereOpt = WhereParser(l).parse(sql) + whereOpt.isSuccess shouldBe true - currentTsLs.foreach { ts => - val parsed = replaceVariable(ts, otherBody) - val json = Json.parse(parsed) + println("=================================================================") + println(sql) + println(whereOpt.get) - (json \ "nextday").as[Long] should be(expectedDayTs) - (json \ "nexthour").as[Long] should be(expectedHourTs) - (json \ "3dayago").as[Long] should be(threeDayAgo) + val ret = whereOpt.get.filter(edge) + if (ret != expected) { + println("==================") + println(s"$whereOpt") + println(s"$edge") + println("==================") } + ret shouldBe expected } - // test("time decay") { - // val ts = System.currentTimeMillis() - // - // for { - // i <- (0 until 10) - // } { - // val timeUnit = 60 * 60 - // val diff = i * timeUnit - // val x = TimeDecay(1.0, 0.05, timeUnit) - // println(x.decay(diff)) - // } - // } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala index 43293d55..efa4d70e 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,29 +24,29 @@ import org.scalatest.{FunSuite, Matchers} class AsynchbaseStorageTest extends FunSuite with Matchers { /** need secured cluster */ -// test("test secure cluster connection") { -// val config = ConfigFactory.parseMap( -// Map( -// "hbase.zookeeper.quorum" -> "localhost", -// "hbase.security.auth.enable" -> "true", -// "hbase.security.authentication" -> "kerberos", -// "hbase.kerberos.regionserver.principal" -> "hbase/_HOST@LOCAL.HBASE", -// "hbase.sasl.clientconfig" -> "Client", -// "java.security.krb5.conf" -> "krb5.conf", -// "java.security.auth.login.config" -> "async-client.jaas.conf") -// ) -// -// val client = AsynchbaseStorage.makeClient(config) -// val table = "test".getBytes() -// -// val putRequest = new PutRequest(table, "a".getBytes(), "e".getBytes, "a".getBytes, "a".getBytes) -// val getRequest = new GetRequest(table, "a".getBytes(), "e".getBytes) -// val ret = client.put(putRequest).join() -// val kvs = client.get(getRequest).join() -// for { -// kv <- kvs -// } { -// println(kv.toString) -// } -// } -} + // test("test secure cluster connection", HBaseTest) { + // val config = ConfigFactory.parseMap( + // Map( + // "hbase.zookeeper.quorum" -> "localhost", + // "hbase.security.auth.enable" -> "true", + // "hbase.security.authentication" -> "kerberos", + // "hbase.kerberos.regionserver.principal" -> "hbase/_HOST@LOCAL.HBASE", + // "hbase.sasl.clientconfig" -> "Client", + // "java.security.krb5.conf" -> "krb5.conf", + // "java.security.auth.login.config" -> "async-client.jaas.conf") + // ) + // + // val client = AsynchbaseStorage.makeClient(config) + // val table = "test".getBytes() + // + // val putRequest = new PutRequest(table, "a".getBytes(), "e".getBytes, "a".getBytes, "a".getBytes) + // val getRequest = new GetRequest(table, "a".getBytes(), "e".getBytes) + // val ret = client.put(putRequest).join() + // val kvs = client.get(getRequest).join() + // for { + // kv <- kvs + // } { + // println(kv.toString) + // } + // } +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala index 5edd5b10..0c585996 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,21 +21,58 @@ package org.apache.s2graph.core.storage.hbase import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta} import org.apache.s2graph.core.types._ -import org.apache.s2graph.core.{IndexEdge, TestCommonWithModels, Vertex} +import org.apache.s2graph.core._ import org.scalatest.{FunSuite, Matchers} -class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() +class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels with TestCommon { + versions map { n => + val ver = s"v$n" + initTests(ver) + val l = label(ver) + + /** note that props have to be properly set up for equals */ + test(s"test serializer/deserializer for index edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withLong(101, ver) + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, + 1.toByte -> InnerVal.withDouble(2.1, ver)) + } + + test(s"test serializer/deserializer for degree edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withStr("0", ver) + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map( + LabelMeta.degreeSeq -> InnerVal.withLong(10, ver), + LabelMeta.timeStampSeq -> tsInnerVal) + + check(ver, l, ts, to, props) + } + + test(s"test serializer/deserializer for incrementCount index edge $ver", HBaseTest) { + val ts = System.currentTimeMillis() + val to = InnerVal.withLong(101, ver) + + val tsInnerVal = InnerVal.withLong(ts, ver) + val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, + 1.toByte -> InnerVal.withDouble(2.1, ver), + LabelMeta.countSeq -> InnerVal.withLong(10, ver)) + + check(ver, l, ts, to, props) + } + } /** * check if storage serializer/deserializer can translate from/to bytes array. - * @param l: label for edge. - * @param ts: timestamp for edge. - * @param to: to VertexId for edge. - * @param props: expected props of edge. + * @param ver: schema version + * @param l: label for edge + * @param ts: timestamp for edge + * @param to: to VertexId for edge + * @param props: expected props of edge */ - def check(l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = { + def check(ver: String, l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = { val from = InnerVal.withLong(1, l.schemaVersion) val vertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, from) val tgtVertexId = TargetVertexId(HBaseType.DEFAULT_COL_ID, to) @@ -44,57 +81,10 @@ class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels { val labelWithDir = LabelWithDirection(l.id.get, 0) val indexEdge = IndexEdge(vertex, tgtVertex, labelWithDir, 0, ts, LabelIndex.DefaultSeq, props) - val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam, + val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam(ver), graph.storage.indexEdgeSerializer(indexEdge).toKeyValues, l.schemaVersion, None) _indexEdgeOpt should not be empty indexEdge should be(_indexEdgeOpt.get) } - - - /** note that props have to be properly set up for equals */ - test("test serializer/deserializer for index edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withLong(101, l.schemaVersion) - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, - 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion)) - - check(l, ts, to, props) - } - } - - test("test serializer/deserializer for degree edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withStr("0", l.schemaVersion) - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map( - LabelMeta.degreeSeq -> InnerVal.withLong(10, l.schemaVersion), - LabelMeta.timeStampSeq -> tsInnerVal) - - check(l, ts, to, props) - } - } - - test("test serializer/deserializer for incrementCount index edge.") { - val ts = System.currentTimeMillis() - for { - l <- Seq(label, labelV2, labelV3, labelV4) - } { - val to = InnerVal.withLong(101, l.schemaVersion) - - val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion) - val props = Map(LabelMeta.timeStampSeq -> tsInnerVal, - 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion), - LabelMeta.countSeq -> InnerVal.withLong(10, l.schemaVersion)) - - check(l, ts, to, props) - } - } -} +} \ No newline at end of file diff --git a/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala index 2f6d0664..0b0c89d9 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,128 +20,132 @@ package org.apache.s2graph.core.types import org.apache.hadoop.hbase.util.Bytes -import org.apache.s2graph.core.TestCommonWithModels +import org.apache.s2graph.core.{CommonTest, TestCommonWithModels} import org.scalatest.{FunSuite, Matchers} class InnerValTest extends FunSuite with Matchers with TestCommonWithModels { - initTests() - import HBaseType.VERSION2 - val decimals = List( - BigDecimal(Long.MinValue), - BigDecimal(Int.MinValue), - BigDecimal(Double.MinValue), - BigDecimal(Float.MinValue), - BigDecimal(Short.MinValue), - BigDecimal(Byte.MinValue), + versions map { n => + val ver = s"v$n" + initTests(ver) - BigDecimal(-1), - BigDecimal(0), - BigDecimal(1), - BigDecimal(Long.MaxValue), - BigDecimal(Int.MaxValue), - BigDecimal(Double.MaxValue), - BigDecimal(Float.MaxValue), - BigDecimal(Short.MaxValue), - BigDecimal(Byte.MaxValue) - ) - val booleans = List( - false, true - ) - val strings = List( - "abc", "abd", "ac", "aca" - ) - val texts = List( - (0 until 1000).map(x => "a").mkString - ) - val blobs = List( - (0 until 1000).map(x => Byte.MaxValue).toArray - ) - def testEncodeDecode(ranges: List[InnerValLike], version: String) = { - for { - innerVal <- ranges - } { - val bytes = innerVal.bytes - val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version) - innerVal == decoded shouldBe true - bytes.length == numOfBytesUsed shouldBe true + + val decimals = List( + BigDecimal(Long.MinValue), + BigDecimal(Int.MinValue), + BigDecimal(Double.MinValue), + BigDecimal(Float.MinValue), + BigDecimal(Short.MinValue), + BigDecimal(Byte.MinValue), + + BigDecimal(-1), + BigDecimal(0), + BigDecimal(1), + BigDecimal(Long.MaxValue), + BigDecimal(Int.MaxValue), + BigDecimal(Double.MaxValue), + BigDecimal(Float.MaxValue), + BigDecimal(Short.MaxValue), + BigDecimal(Byte.MaxValue) + ) + val booleans = List( + false, true + ) + val strings = List( + "abc", "abd", "ac", "aca" + ) + val texts = List( + (0 until 1000).map(x => "a").mkString + ) + val blobs = List( + (0 until 1000).map(x => Byte.MaxValue).toArray + ) + def testEncodeDecode(ranges: List[InnerValLike], version: String) = { + for { + innerVal <- ranges + } { + val bytes = innerVal.bytes + val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version) + innerVal == decoded shouldBe true + bytes.length == numOfBytesUsed shouldBe true + } } - } - // test("big decimal") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)} - // testEncodeDecode(innerVals, version) - // } - // } - // test("text") { - // for { - // version <- List(VERSION2) - // } { - // val innerVals = texts.map { t => InnerVal.withStr(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("string") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = strings.map { t => InnerVal.withStr(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("blob") { - // for { - // version <- List(VERSION2) - // } { - // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - // test("boolean") { - // for { - // version <- List(VERSION2, VERSION1) - // } { - // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) } - // testEncodeDecode(innerVals, version) - // } - // } - test("korean") { - val small = InnerVal.withStr("가", VERSION2) - val large = InnerVal.withStr("나", VERSION2) - val smallBytes = small.bytes - val largeBytes = large.bytes + // test("big decimal") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)} + // testEncodeDecode(innerVals, version) + // } + // } + // test("text") { + // for { + // version <- List(VERSION2) + // } { + // val innerVals = texts.map { t => InnerVal.withStr(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("string") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = strings.map { t => InnerVal.withStr(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("blob") { + // for { + // version <- List(VERSION2) + // } { + // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + // test("boolean") { + // for { + // version <- List(VERSION2, VERSION1) + // } { + // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) } + // testEncodeDecode(innerVals, version) + // } + // } + test(s"korean $ver", CommonTest) { + val small = InnerVal.withStr("가", ver) + val large = InnerVal.withStr("나", ver) + val smallBytes = small.bytes + val largeBytes = large.bytes - println (Bytes.compareTo(smallBytes, largeBytes)) - true + println (Bytes.compareTo(smallBytes, largeBytes)) + true + } + // test("innerVal") { + // val srcVal = InnerVal.withLong(44391298, VERSION2) + // val srcValV1 = InnerVal.withLong(44391298, VERSION1) + // val tgtVal = InnerVal.withLong(7295564, VERSION2) + // + // val a = VertexId(0, srcVal) + // val b = SourceVertexId(0, srcVal) + // val c = TargetVertexId(0, srcVal) + // val aa = VertexId(0, srcValV1) + // val bb = SourceVertexId(0, srcValV1) + // val cc = TargetVertexId(0, srcValV1) + // println(a.bytes.toList) + // println(b.bytes.toList) + // println(c.bytes.toList) + // + // println(aa.bytes.toList) + // println(bb.bytes.toList) + // println(cc.bytes.toList) + // } + // test("aa") { + // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes + // val pbr = new SimplePositionedByteRange(bytes) + // pbr.setOffset(1) + // println(pbr.getPosition) + // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr) + // println(pbr.getPosition) + // true + // } } - // test("innerVal") { - // val srcVal = InnerVal.withLong(44391298, VERSION2) - // val srcValV1 = InnerVal.withLong(44391298, VERSION1) - // val tgtVal = InnerVal.withLong(7295564, VERSION2) - // - // val a = VertexId(0, srcVal) - // val b = SourceVertexId(0, srcVal) - // val c = TargetVertexId(0, srcVal) - // val aa = VertexId(0, srcValV1) - // val bb = SourceVertexId(0, srcValV1) - // val cc = TargetVertexId(0, srcValV1) - // println(a.bytes.toList) - // println(b.bytes.toList) - // println(c.bytes.toList) - // - // println(aa.bytes.toList) - // println(bb.bytes.toList) - // println(cc.bytes.toList) - // } - // test("aa") { - // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes - // val pbr = new SimplePositionedByteRange(bytes) - // pbr.setOffset(1) - // println(pbr.getPosition) - // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr) - // println(pbr.getPosition) - // true - // } -} +} \ No newline at end of file