Notification System Fixes - Changes Route

Makefile

@@ -80,36 +80,24 @@ docker: .docker-exec
 	docker network create $(DOCKER_NETWORK)
 	@touch $@
 
-# Using dot files to hold the modification time the docker image and container were built
-#.docker-bld: .docker-network
-#	docker build -t $(image-string):bld $(DOCKER_OPTS) -f Dockerfile-bld --build-arg SCALA_VERSION=$(SCALA_VERSION) .
-#	- docker rm -f $(DOCKER_NAME)_bld 2> /dev/null || :
-#	docker run --name $(DOCKER_NAME)_bld --network $(DOCKER_NETWORK) -d -t -v $(CURDIR):$(EXCHANGE_API_DIR) $(image-string):bld /bin/bash
-#	@touch $@
-
-#.docker-compile: $(wildcard src/main/scala/com/horizon/exchangeapi/*) $(wildcard src/main/resources/*) .docker-bld
-#	docker exec -t $(DOCKER_NAME)_bld /bin/bash -c "cd $(EXCHANGE_API_DIR) && sbt package"
-#	# war file ends up in: ./target/scala-$SCALA_VERSION_SHORT/exchange-api_$SCALA_VERSION_SHORT-$EXCHANGE_API_WAR_VERSION.war
-#	@touch $@
-
-.docker-exec: .docker-network
+.docker-exec:
 	sbt docker:publishLocal
 	@touch $@
 
-.docker-exec-run: .docker-exec start-docker-exec
-	@touch $@
-
-# Start the already built image (w/o building 1st)
-start-docker-exec: .docker-network
+.docker-exec-run: .docker-exec .docker-network
 	@if [[ ! -f "$(EXCHANGE_HOST_KEYSTORE_DIR)/keystore" || ! -f "$(EXCHANGE_HOST_KEYSTORE_DIR)/keypassword" ]]; then echo "Error: keystore and keypassword do not exist in $(EXCHANGE_HOST_KEYSTORE_DIR). You must first copy them there or run 'make gen-key'"; false; fi
 	- docker rm -f $(DOCKER_NAME) 2> /dev/null || :
-#	docker run --name $(DOCKER_NAME) --network $(DOCKER_NETWORK) -d -t -p $(EXCHANGE_API_PORT):$(EXCHANGE_API_PORT) -p $(EXCHANGE_API_HTTPS_PORT):$(EXCHANGE_API_HTTPS_PORT) -e "ICP_EXTERNAL_MGMT_INGRESS=$$ICP_EXTERNAL_MGMT_INGRESS" -v $(EXCHANGE_HOST_CONFIG_DIR):$(EXCHANGE_CONFIG_DIR) -v $(EXCHANGE_HOST_ICP_CERT_FILE):$(EXCHANGE_ICP_CERT_FILE) -v $(EXCHANGE_HOST_KEYSTORE_DIR):$(EXCHANGE_CONTAINER_KEYSTORE_DIR):ro -v $(EXCHANGE_HOST_POSTGRES_CERT_FILE):$(EXCHANGE_CONTAINER_POSTGRES_CERT_FILE) $(image-string):$(DOCKER_TAG)
+	docker run --name $(DOCKER_NAME) --network $(DOCKER_NETWORK) -d -t -p $(EXCHANGE_API_PORT):$(EXCHANGE_API_PORT) -p $(EXCHANGE_API_HTTPS_PORT):$(EXCHANGE_API_HTTPS_PORT) -e "ICP_EXTERNAL_MGMT_INGRESS=$$ICP_EXTERNAL_MGMT_INGRESS" -v $(EXCHANGE_HOST_CONFIG_DIR):$(EXCHANGE_CONFIG_DIR) -v $(EXCHANGE_HOST_ICP_CERT_FILE):$(EXCHANGE_ICP_CERT_FILE) -v $(EXCHANGE_HOST_KEYSTORE_DIR):$(EXCHANGE_CONTAINER_KEYSTORE_DIR):ro -v $(EXCHANGE_HOST_POSTGRES_CERT_FILE):$(EXCHANGE_CONTAINER_POSTGRES_CERT_FILE) $(image-string):$(DOCKER_TAG)
+	@touch $@
 
-start-docker-exec-no-https: .docker-network
+# Note: this target is used by travis as part of testing
+.docker-exec-run-no-https: .docker-exec .docker-network
 	- docker rm -f $(DOCKER_NAME) 2> /dev/null || :
 	docker run --name $(DOCKER_NAME) --network $(DOCKER_NETWORK) -d -t -p $(EXCHANGE_API_PORT):$(EXCHANGE_API_PORT) -v $(EXCHANGE_HOST_CONFIG_DIR):$(EXCHANGE_CONFIG_DIR) $(image-string):$(DOCKER_TAG)
+	@touch $@
 
 # Run the automated tests in the bld container against the exchange svr running in the exec container
+# Note: this target is used by travis as part of testing
 test: .docker-bld
 	: $${EXCHANGE_ROOTPW:?}   # this verifies these env vars are set
 	docker exec -t \
@@ -180,4 +168,4 @@ version:
 
 .SECONDARY:
 
-.PHONY: default clean clean-exec-image clean-all start-docker-exec-no-https start-docker-exec docker test docker-push-only docker-push-version-only docker-push docker-push-to-prod gen-key sync-swagger-ui testmake version
+.PHONY: default clean clean-exec-image clean-all docker test docker-push-only docker-push-version-only docker-push docker-push-to-prod gen-key sync-swagger-ui testmake version

README.md

@@ -54,8 +54,8 @@ make gen-key
 
 - `sbt`
 - `~reStart`
-- Once the server starts, to try a simple rest method browse: [http://localhost:8080/v1/admin/version](http://localhost:8080/v1/admin/version)
-- To see the swagger output, browse: [http://localhost:8080/api](http://localhost:8080/api)
+- Once the server starts, to see the swagger output, browse: [http://localhost:8080/v1/swagger](http://localhost:8080/v1/swagger)
+- To try a simple rest method curl: `curl -X GET "http://localhost:8080/v1/admin/version"`. You should get the exchange version number as the response.  
 - A convenience script `src/test/bash/primedb.sh` can be run to prime the DB with some exchange resources to use in manually testing:
 ```
 export EXCHANGE_USER=<my-user-in-IBM-org>
@@ -220,6 +220,11 @@ Now you can disable root by setting `api.root.enabled` to `false` in `/etc/horiz
     - detect if a pattern is updated with service that has userInput w/o default values, and give warning
     - Consider changing all creates to POST, and update (via put/patch) return codes to 200
 
+## Changes in 2.2.0
+
+- Issue 258: Notification Framework bugfixes
+- Issue 265: POST /v1/orgs/{orgid}/search/nodes/error now filters on orgid 
+
 ## Changes in 2.1.0
 
 - Added `heartbeatIntervals` field to org and node resources

src/main/resources/version.txt

@@ -1 +1 @@
-2.1.0
+2.2.0

src/main/scala/com/horizon/exchangeapi/OrgsRoutes.scala

@@ -126,7 +126,7 @@ final case class ChangeEntry(orgId: String, var resource: String, id: String, va
   def setOperation(newOp: String) {this.operation = newOp}
   def setResource(newResource: String) {this.resource = newResource}
 }
-final case class ResourceChangesRespObject(changes: List[ChangeEntry], mostRecentChangeId: Int, exchangeVersion: String)
+final case class ResourceChangesRespObject(changes: List[ChangeEntry], mostRecentChangeId: Int, maxChangeIdOfQuery: Int, exchangeVersion: String)
 
 /** Routes for /orgs */
 @Path("/v1/orgs")
@@ -419,8 +419,8 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
     exchAuth(TNode(OrgAndId(orgid,"*").toString),Access.READ) { _ =>
       complete({
         val q = for {
-          (n) <- NodeErrorTQ.rows.filter(_.errors =!= "").filter(_.errors =!= "[]")
-        } yield n.nodeId
+          (n, _) <- NodesTQ.rows.filter(_.orgid === orgid) join NodeErrorTQ.rows.filter(_.errors =!= "").filter(_.errors =!= "[]") on (_.id === _.nodeId)
+        } yield n.id
 
         db.run(q.result).map({ list =>
           logger.debug("POST /orgs/"+orgid+"/search/nodes/error result size: "+list.size)
@@ -526,6 +526,7 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
     val changesList = ListBuffer[ChangeEntry]()
     var mostRecentChangeId = 0
     var entryCounter = 0
+    var maxChangeIdOfQuery = 0
     breakable {
       for(input <- inputList) { //this for loop should only ever be of size 2
         val changesMap = scala.collection.mutable.Map[String, ChangeEntry]() //using a Map allows us to avoid having a loop in a loop when searching the map for the resource id
@@ -544,18 +545,19 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
             }
            */
           val resChange = ResourceChangesInnerObject(entry._1, entry._8)
-          if(changesMap.isDefinedAt(entry._3)){  // using the map allows for better searching and entry
-            if(changesMap(entry._3).resourceChanges.last.changeId < entry._1){
+          if(changesMap.isDefinedAt(entry._3+"_"+entry._6)){  // using the map allows for better searching and entry
+            if(changesMap(entry._3+"_"+entry._6).resourceChanges.last.changeId < entry._1){
               // the entry we are looking at actually happened later than the last entry in resourceChanges
               // doing this check by changeId on the off chance two changes happen at the exact same time changeId tells which one is most updated
-              changesMap(entry._3).addToResourceChanges(resChange) // add the changeId and lastUpdated to the list of recent changes
-              changesMap(entry._3).setOperation(entry._7) // update the most recent operation performed
-              changesMap(entry._3).setResource(entry._6) // update exactly what resource was most recently touched
+              changesMap(entry._3+"_"+entry._6).addToResourceChanges(resChange) // add the changeId and lastUpdated to the list of recent changes
+              changesMap(entry._3+"_"+entry._6).setOperation(entry._7) // update the most recent operation performed
             }
           } else{
             val resChangeListBuffer = ListBuffer[ResourceChangesInnerObject](resChange)
-            changesMap(entry._3) = ChangeEntry(entry._2, entry._6, entry._3, entry._7, resChangeListBuffer)
+            changesMap(entry._3+"_"+entry._6) = ChangeEntry(entry._2, entry._6, entry._3, entry._7, resChangeListBuffer)
           }
+          //check maxChangeIdOfQuery
+          if (entry._1 > maxChangeIdOfQuery) {maxChangeIdOfQuery = entry._1}
         }
         // convert changesMap to ListBuffer[ChangeEntry]
         breakable {
@@ -569,7 +571,7 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
         if (entryCounter > maxResp) break // if we are over the count of allowed entries just stop and return the list as is
       }
     }
-    ResourceChangesRespObject(changesList.toList, mostRecentChangeId, exchangeVersion)
+    ResourceChangesRespObject(changesList.toList, mostRecentChangeId, maxChangeIdOfQuery, exchangeVersion)
   }
 
   /* ====== POST /orgs/{orgid}/changes ================================ */
@@ -605,22 +607,22 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
             r <- ResourceChangesTQ.rows.filter(_.orgId === orgId).filter(_.lastUpdated >= lastTime).filter(_.changeId >= reqBody.changeId)
           } yield (r.changeId, r.orgId, r.id, r.category, r.public, r.resource, r.operation, r.lastUpdated)
 
-          val qIBM = for {
-            r <- ResourceChangesTQ.rows.filter(_.orgId === "IBM").filter(_.public === "true").filter(_.lastUpdated >= lastTime).filter(_.changeId >= reqBody.changeId)
+          val qPublic = for {
+            r <- ResourceChangesTQ.rows.filter(_.orgId =!= orgId).filter(_.public === "true").filter(_.lastUpdated >= lastTime).filter(_.changeId >= reqBody.changeId)
           } yield (r.changeId, r.orgId, r.id, r.category, r.public, r.resource, r.operation, r.lastUpdated)
 
           var qOrgResp : scala.Seq[(Int, String, String, String, String, String, String, String)] = null
-          var qIBMResp : scala.Seq[(Int, String, String, String, String, String, String, String)] = null
+          var qPublicResp : scala.Seq[(Int, String, String, String, String, String, String, String)] = null
 
           db.run(qOrg.result.asTry.flatMap({
             case Success(qOrgResult) =>
               //logger.debug("POST /orgs/" + orgId + "/changes changes in caller org: " + qOrgResult.toString())
               logger.debug("POST /orgs/" + orgId + "/changes changes in caller org: " + qOrgResult.size)
               qOrgResp = qOrgResult
-              qIBM.result.asTry
+              qPublic.result.asTry
             case Failure(t) => DBIO.failed(t).asTry
           }).flatMap({
-            case Success(qIBMResult) => qIBMResp = qIBMResult
+            case Success(qIBMResult) => qPublicResp = qIBMResult
               //logger.debug("POST /orgs/" + orgId + "/changes public changes in IBM org: " + qIBMResult.toString())
               logger.debug("POST /orgs/" + orgId + "/changes public changes in IBM org: " + qIBMResult.size)
               val id = orgId + "/" + ident.getIdentity
@@ -638,7 +640,7 @@ class OrgsRoutes(implicit val system: ActorSystem) extends JacksonSupport with A
           })).map({
             case Success(n) =>
               logger.debug(s"POST /orgs/$orgId result: $n")
-              if (n > 0) (HttpCode.POST_OK, buildResourceChangesResponse(qOrgResp, qIBMResp, reqBody.maxRecords))
+              if (n > 0) (HttpCode.POST_OK, buildResourceChangesResponse(qOrgResp, qPublicResp, reqBody.maxRecords))
               else (HttpCode.NOT_FOUND, ApiResponse(ApiRespType.NOT_FOUND, ExchMsg.translate("node.or.agbot.not.found", ident.getIdentity)))
             case Failure(t) =>
               (HttpCode.BAD_INPUT, ApiResponse(ApiRespType.BAD_INPUT, ExchMsg.translate("invalid.input.message", t.getMessage)))

src/test/scala/exchangeapi/NodesSuite.scala

@@ -314,6 +314,7 @@ class NodesSuite extends FunSuite {
     assert(parsedBody.changes.exists(y => {(y.id == nodeId) && (y.operation == ResourceChangeConfig.CREATEDMODIFIED) && (y.resource == "node") && (y.resourceChanges.size == 1)}))
     assert(parsedBody.changes.size <= maxRecords)
     assert(parsedBody.mostRecentChangeId != 0)
+    assert(parsedBody.maxChangeIdOfQuery != 0)
     assert(parsedBody.exchangeVersion == ExchangeApiAppMethods.adminVersion().toString)
   }
 
@@ -2322,21 +2323,30 @@ class NodesSuite extends FunSuite {
     assert(response.body.contains("update all but token"))
   }
 
+  test("PUT /orgs/"+orgid+"/nodes/"+nodeId+"/errors - add an error to later grab it from the changes route") {
+    val input = """{ "errors": [{ "record_id":"1", "message":"test error 1", "event_code":"500", "hidden":false, "workload":{"url":"myservice"}, "timestamp":"yesterday" }] }"""
+    val response = Http(URL+"/nodes/"+nodeId+"/errors").postData(input).method("put").headers(CONTENT).headers(ACCEPT).headers(NODEAUTH).asString
+    info("POST DATA: " + write(input))
+    info("code: "+response.code+", response.body: "+response.body)
+    assert(response.code === HttpCode.PUT_OK.intValue)
+  }
+
   test("DELETE /orgs/"+orgid+"/nodes/"+nodeId) {
     val response = Http(URL+"/nodes/"+nodeId).method("delete").headers(CONTENT).headers(ACCEPT).headers(ROOTAUTH).asString
     info("code: "+response.code+", response.body: "+response.body)
     assert(response.code === HttpCode.DELETED.intValue)
   }
 
-  test("POST /orgs/"+orgid+"/changes - verify " + nodeId + " was deleted and logged as deleted") {
+  test("POST /orgs/"+orgid+"/changes - verify " + nodeId + " was deleted and logged as deleted also that node error change is there") {
     val time = ApiTime.pastUTC(60)
     val input = ResourceChangesRequest(0, Some(time), 1000, None)
     val response = Http(URL+"/changes").postData(write(input)).method("post").headers(CONTENT).headers(ACCEPT).headers(ROOTAUTH).asString
     info("code: "+response.code)
     assert(response.code === HttpCode.POST_OK.intValue)
     assert(!response.body.isEmpty)
     val parsedBody = parse(response.body).extract[ResourceChangesRespObject]
-    assert(parsedBody.changes.exists(y => {(y.id == nodeId) && (y.operation == ResourceChangeConfig.DELETED)}))
+    assert(parsedBody.changes.exists(y => {(y.id == nodeId) && (y.operation == ResourceChangeConfig.DELETED) && (y.resource == "node")}))
+    assert(parsedBody.changes.exists(y => {(y.id == nodeId) && (y.operation == ResourceChangeConfig.CREATEDMODIFIED) && (y.resource == "nodeerrors")}))
   }
 
   test("DELETE /orgs/"+orgid+"/nodes/"+nodeId + " try to delete again -- should fail") {

src/test/scala/exchangeapi/ServicesSuite.scala

@@ -33,8 +33,11 @@ class ServicesSuite extends FunSuite {
   val CONTENT = ("Content-Type","application/json")
   val CONTENTTEXT = ("Content-Type","text/plain")
   val orgid = "ServicesSuiteTests"
+  val orgid2 = "ServicesSuiteTests-SecondOrg"
   val authpref=orgid+"/"
+  val authpref2=orgid2+"/"
   val URL = urlRoot+"/v1/orgs/"+orgid
+  val URL2 = urlRoot+"/v1/orgs/"+orgid2
   val user = "9999"
   val orguser = authpref+user
   val pw = user+"pw"
@@ -62,6 +65,7 @@ class ServicesSuite extends FunSuite {
   val svcArch = "arm"
   val service = svcBase + "_" + svcVersion + "_" + svcArch
   val orgservice = authpref+service
+  val org2service = authpref2+service
   val svcBase2 = "svc9921"
   val svcUrl2 = "http://" + svcBase2
   val svcVersion2 = "1.0.0"
@@ -137,6 +141,19 @@ class ServicesSuite extends FunSuite {
     assert(response.code === HttpCode.POST_OK.intValue)
   }
 
+  /** Create a second org to use for this test */
+  test("POST /orgs/"+orgid2+" - create org") {
+    // Try deleting it 1st, in case it is left over from previous test
+    var response = Http(URL2).method("delete").headers(ACCEPT).headers(ROOTAUTH).asString
+    info("code: "+response.code+", response.body: "+response.body)
+    assert(response.code === HttpCode.DELETED.intValue || response.code === HttpCode.NOT_FOUND.intValue)
+
+    val input = PostPutOrgRequest(None, "My Second Org", "desc", None, None)
+    response = Http(URL2).postData(write(input)).method("post").headers(CONTENT).headers(ACCEPT).headers(ROOTAUTH).asString
+    info("code: "+response.code+", response.body: "+response.body)
+    assert(response.code === HttpCode.POST_OK.intValue)
+  }
+
   /** Delete all the test users, in case they exist from a previous run. Do not need to delete the services, because they are deleted when the user is deleted. */
   test("Begin - DELETE all test users") {
     if (rootpw == "") fail("The exchange root password must be set in EXCHANGE_ROOTPW and must also be put in config.json.")
@@ -224,6 +241,15 @@ class ServicesSuite extends FunSuite {
     assert(respObj.msg.contains("service '"+orgservice+"' created"))
   }
 
+  test("POST /orgs/"+orgid2+"/services - add public "+service+" as root in second org to check that its in response") {
+    val input = PostPutServiceRequest(svcBase+" arm", None, public = true, Some(svcDoc), svcUrl, svcVersion, svcArch, "multiple", None, None, Some(List(Map("name" -> "foo"))), "{\"services\":{}}","a",None)
+    val response = Http(URL2+"/services").postData(write(input)).method("post").headers(CONTENT).headers(ACCEPT).headers(ROOTAUTH).asString
+    info("code: "+response.code+", response.body: "+response.body)
+    assert(response.code === HttpCode.POST_OK.intValue)
+    val respObj = parse(response.body).extract[ApiResponse]
+    assert(respObj.msg.contains("service '"+org2service+"' created"))
+  }
+
   test("POST /orgs/"+orgid+"/changes - verify " + service + " was created and stored") {
     val time = ApiTime.pastUTC(secondsAgo)
     val input = ResourceChangesRequest(0, Some(time), maxRecords, None)
@@ -232,7 +258,8 @@ class ServicesSuite extends FunSuite {
     assert(response.code === HttpCode.POST_OK.intValue)
     assert(!response.body.isEmpty)
     val parsedBody = parse(response.body).extract[ResourceChangesRespObject]
-    assert(parsedBody.changes.exists(y => {(y.id == service) && (y.operation == ResourceChangeConfig.CREATED) && (y.resource == "service")}))
+    assert(parsedBody.changes.exists(y => {(y.orgId == orgid) && (y.id == service) && (y.operation == ResourceChangeConfig.CREATED) && (y.resource == "service")}))
+    assert(parsedBody.changes.exists(y => {(y.orgId == orgid2) && (y.id == service) && (y.operation == ResourceChangeConfig.CREATED) && (y.resource == "service")}))
   }
 
   test("POST /orgs/"+orgid+"/services - add "+service3+" as user that requires a service with reqService.versionRange and version") {