Add ms workloads

Makefile

@@ -5,7 +5,7 @@
 SHELL = /bin/bash -e
 ARCH ?= x86
 DOCKER_NAME ?= exchange-api
-DOCKER_TAG ?= v1.24.0
+DOCKER_TAG ?= v1.25.0
 DOCKER_OPTS ?= --no-cache
 COMPILE_CLEAN ?= clean
 image-string = $(DOCKER_REGISTRY)/$(ARCH)/exchange-api
@@ -39,7 +39,7 @@ clean: clean-exec-image
 
 clean-exec-image:
 	- docker rm -f $(DOCKER_NAME) 2> /dev/null || :
-	- docker rmi $(image-string):{$(DOCKER_TAG),latest} 2> /dev/null || :
+	- docker rmi $(image-string):{$(DOCKER_TAG),volcanostaging} 2> /dev/null || :
 	rm -f .docker-exec .docker-exec-run
 
 # Also remove the bld image/container
@@ -67,15 +67,15 @@ docker: .docker-exec
 
 .docker-exec: .docker-compile
 	docker build -t $(image-string):$(DOCKER_TAG) $(DOCKER_OPTS) -f Dockerfile-exec --build-arg SCALA_VERSION=$(SCALA_VERSION) --build-arg SCALA_VERSION_SHORT=$(SCALA_VERSION_SHORT) --build-arg EXCHANGE_API_WAR_VERSION=$(EXCHANGE_API_WAR_VERSION) .
-	docker tag $(image-string):$(DOCKER_TAG) $(image-string):latest
+	docker tag $(image-string):$(DOCKER_TAG) $(image-string):volcanostaging
 	@touch $@
 
 # rem-docker-exec:
 # 	- docker rm -f $(DOCKER_NAME) 2> /dev/null || :
 
 .docker-exec-run: .docker-exec
 	- docker rm -f $(DOCKER_NAME) 2> /dev/null || :
-	docker run --name $(DOCKER_NAME) -d -t -p $(EXCHANGE_API_PORT):$(EXCHANGE_API_PORT) -v $(EXCHANGE_HOST_CONFIG_DIR):$(EXCHANGE_CONFIG_DIR) $(image-string):latest
+	docker run --name $(DOCKER_NAME) -d -t -p $(EXCHANGE_API_PORT):$(EXCHANGE_API_PORT) -v $(EXCHANGE_HOST_CONFIG_DIR):$(EXCHANGE_CONFIG_DIR) $(image-string):$(DOCKER_TAG)
 	@touch $@
 
 # Run the automated tests in the bld container against the exchange svr running in the exec container
@@ -86,9 +86,9 @@ docker: .docker-exec
 # Push the docker images to the registry w/o rebuilding them
 docker-push-only:
 	docker push $(image-string):$(DOCKER_TAG)
-	docker push $(image-string):latest
+	docker push $(image-string):volcanostaging
 
-# Push the image with the explicit version tag (so someone else can test it), but do not push the latest tag so it does not get deployed to stg/prod yet
+# Push the image with the explicit version tag (so someone else can test it), but do not push the volcanostaging tag so it does not get deployed to stg yet
 docker-push-version-only:
 	docker push $(image-string):$(DOCKER_TAG)
 

README.md

@@ -50,6 +50,7 @@ services in the exchange.
 - Once the server starts, to try a simple rest method browse: [http://localhost:8080/v1/devices?id=a&token=b](http://localhost:8080/v1/devices?id=a&token=b)
 - To see the swagger output, browse: [http://localhost:8080/api](http://localhost:8080/api)
 - Run the automated tests (with the exchange server still running): `./sbt test`
+- Run just 1 of the the automated test suites (with the exchange server still running): `./sbt "test-only exchangeapi.AgbotsSuite"`
 - Run the performance tests: `src/test/bash/scale/test.sh` or `src/test/bash/scale/wrapper.sh 8`
 
 ## Building and Running the Container
@@ -75,6 +76,12 @@ services in the exchange.
 
 - The properties used for advertising and searching should always have its value element be a string. For example the memory property value should be `"300"` instead of `300`. This is because scalatra is automatically converting the json to scala data structures, and i don't know how to have the data structures that vary in type.
 
+## Changes Between v1.24.0 and v1.25.0
+
+- Added /microservices and /workloads resources
+- Changed default permissions to allow devices and agbots to read all agbots
+- Changed additional image tag from latest to volcanostaging
+
 ## Changes Between v1.23.0 and v1.24.0
 
 - Converted to the newer build.sbt file and upgraded to sbt version 0.13.13

build.sbt

@@ -15,7 +15,8 @@ lazy val root = (project in file(".")).
       "com.typesafe.slick" %% "slick-hikaricp" % "latest.integration",
       "org.postgresql" % "postgresql" % "latest.integration",
       "com.zaxxer" % "HikariCP" % "latest.integration",
-      "org.slf4j" % "slf4j-api" % "latest.integration",
+      // "org.slf4j" % "slf4j-api" % "latest.release",  // they put version 1.8.0-alpha2 prematurely into latest.release
+      "org.slf4j" % "slf4j-api" % "1.7.+",
       "ch.qos.logback" % "logback-classic" % "latest.integration",
       "com.mchange" % "c3p0" % "latest.integration",
       "javax.servlet" % "javax.servlet-api" % "latest.integration" % "provided",

src/main/resources/config.json

@@ -8,7 +8,9 @@
 			"maxAgbots": 1000,		// maximum number of agbots 1 user is allowed to create
 			"maxAgreements": 1000,	// maximum number of agreements 1 device or agbot is allowed to create
 			"maxMessagesInMailbox": 500,	// maximum number of msgs currently in 1 device or agbot mailbox (the sending side is handled by rate limiting)
-			"maxBlockchains": 100		// maximum number of BC types or instance definitions 1 user is allowed to create
+			"maxBlockchains": 100,		// maximum number of BC types or instance definitions 1 user is allowed to create
+			"maxMicroservices": 100		// maximum number of microservices 1 user is allowed to create
+			"maxWorkloads": 100		// maximum number of workloads 1 user is allowed to create
 		},
 		"specRef": {
 			"prefix": "https://bluehorizon.network/documentation/",
@@ -46,10 +48,10 @@
 		},
 		"acls": {
 			"ANONYMOUS": ["CREATE_USER", "RESET_USER_PW"],
-			"USER": ["READ_MYSELF", "WRITE_MYSELF", "RESET_USER_PW", "CREATE_DEVICE", "READ_MY_DEVICES", "WRITE_MY_DEVICES", "READ_ALL_DEVICES", "CREATE_AGBOT", "READ_MY_AGBOTS", "WRITE_MY_AGBOTS", "DATA_HEARTBEAT_MY_AGBOTS", "READ_ALL_AGBOTS", "STATUS", "READ_MY_BLOCKCHAINS", "READ_ALL_BLOCKCHAINS", "WRITE_MY_BLOCKCHAINS", "CREATE_BLOCKCHAINS", "READ_MY_BCTYPES", "READ_ALL_BCTYPES", "WRITE_MY_BCTYPES", "CREATE_BCTYPES"],
+			"USER": ["READ_MYSELF", "WRITE_MYSELF", "RESET_USER_PW", "CREATE_DEVICE", "READ_MY_DEVICES", "WRITE_MY_DEVICES", "READ_ALL_DEVICES", "CREATE_AGBOT", "READ_MY_AGBOTS", "WRITE_MY_AGBOTS", "DATA_HEARTBEAT_MY_AGBOTS", "READ_ALL_AGBOTS", "STATUS", "READ_MY_BLOCKCHAINS", "READ_ALL_BLOCKCHAINS", "WRITE_MY_BLOCKCHAINS", "CREATE_BLOCKCHAINS", "READ_MY_BCTYPES", "READ_ALL_BCTYPES", "WRITE_MY_BCTYPES", "CREATE_BCTYPES", "READ_MY_MICROSERVICES", "READ_ALL_MICROSERVICES", "WRITE_MY_MICROSERVICES", "CREATE_MICROSERVICES", "READ_MY_WORKLOADS", "READ_ALL_WORKLOADS", "WRITE_MY_WORKLOADS", "CREATE_WORKLOADS"],
 			"SUPERUSER": ["ALL"],
-			"DEVICE": ["READ_MYSELF", "WRITE_MYSELF", "READ_MY_DEVICES", "SEND_MSG_TO_AGBOT", "READ_ALL_BLOCKCHAINS", "READ_ALL_BCTYPES"],
-			"AGBOT": ["READ_MYSELF", "WRITE_MYSELF", "DATA_HEARTBEAT_MY_AGBOTS", "READ_MY_AGBOTS", "READ_ALL_DEVICES", "SEND_MSG_TO_DEVICE", "READ_ALL_BLOCKCHAINS", "READ_ALL_BCTYPES"]
+			"DEVICE": ["READ_MYSELF", "WRITE_MYSELF", "READ_MY_DEVICES", "SEND_MSG_TO_AGBOT", "READ_ALL_AGBOTS", "READ_ALL_BLOCKCHAINS", "READ_ALL_BCTYPES", "READ_ALL_MICROSERVICES", "READ_ALL_WORKLOADS"],
+			"AGBOT": ["READ_MYSELF", "WRITE_MYSELF", "DATA_HEARTBEAT_MY_AGBOTS", "READ_MY_AGBOTS", "READ_ALL_AGBOTS", "READ_ALL_DEVICES", "SEND_MSG_TO_DEVICE", "READ_ALL_BLOCKCHAINS", "READ_ALL_BCTYPES", "READ_ALL_MICROSERVICES", "READ_ALL_WORKLOADS"]
 		}
 	}
 }

src/main/scala/com/horizon/exchangeapi/ApiUtils.scala

@@ -133,6 +133,7 @@ object HttpCode {
   val BAD_INPUT = 400     // invalid user input, usually in the params or json body
   val BADCREDS = 401    // user/pw or id/token is wrong (they call it unauthorized, but it is really unauthenticated)
   val ACCESS_DENIED = 403   // do not have authorization to access this resource
+  val ALREADY_EXISTS = 403   // trying to create a resource that already exists. For now using 403 (forbidden), but could also use 409 (conflict)
   val NOT_FOUND = 404   // resource not found
   val INTERNAL_ERROR = 500
   val NOT_IMPLEMENTED = 501
@@ -143,6 +144,7 @@ case class ApiResponse(code: String, msg: String)
 object ApiResponseType {
   val BADCREDS = "invalid-credentials"
   val ACCESS_DENIED = "access-denied"
+  val ALREADY_EXISTS = "already-exists"
   val BAD_INPUT = "invalid-input"
   val NOT_FOUND = "not-found"
   val INTERNAL_ERROR = "internal_error"

src/main/scala/com/horizon/exchangeapi/AuthenticationSupport.scala

@@ -52,6 +52,16 @@ object Access extends Enumeration {
   val READ_ALL_BCTYPES = Value("READ_ALL_BCTYPES")
   val WRITE_ALL_BCTYPES = Value("WRITE_ALL_BCTYPES")
   val CREATE_BCTYPES = Value("CREATE_BCTYPES")       // we use WRITE_MY_BCTYPES instead of this
+  val READ_MY_MICROSERVICES = Value("READ_MY_MICROSERVICES")
+  val WRITE_MY_MICROSERVICES = Value("WRITE_MY_MICROSERVICES")
+  val READ_ALL_MICROSERVICES = Value("READ_ALL_MICROSERVICES")
+  val WRITE_ALL_MICROSERVICES = Value("WRITE_ALL_MICROSERVICES")
+  val CREATE_MICROSERVICES = Value("CREATE_MICROSERVICES")
+  val READ_MY_WORKLOADS = Value("READ_MY_WORKLOADS")
+  val WRITE_MY_WORKLOADS = Value("WRITE_MY_WORKLOADS")
+  val READ_ALL_WORKLOADS = Value("READ_ALL_WORKLOADS")
+  val WRITE_ALL_WORKLOADS = Value("WRITE_ALL_WORKLOADS")
+  val CREATE_WORKLOADS = Value("CREATE_WORKLOADS")
   val ADMIN = Value("ADMIN")
   val STATUS = Value("STATUS")
   val ALL = Value("ALL")
@@ -121,7 +131,7 @@ case class Creds(id: String, token: String) {     // id and token are generic na
 object AuthCache {
   val logger = LoggerFactory.getLogger(ExchConfig.LOGGER)
 
-  /** 1 set of things (user/pw, device id/token, or agbot id/token) */
+  /** 1 set of things (user/pw, device id/token, agbot id/token, bctype/owner, bc/owner, microservice/owner, workload/owner) */
   class Cache(val whichTab: String) {     //TODO: i am sure there is a better way to handle the different tables
     // Throughout the implementation of this class, id and token are used generically, meaning in the case of users they are user and pw.
     // Our goal is for the token to be unhashed, but we have to handle the case where the user gives us an already hashed token.
@@ -146,6 +156,8 @@ object AuthCache {
         case "agbots" => db.run(AgbotsTQ.rows.map(x => (x.id, x.token, x.owner)).result).map({ list => this._initIds(list) })
         case "bctypes" => db.run(BctypesTQ.rows.map(x => (x.bctype, x.definedBy)).result).map({ list => this._initBctypes(list) })
         case "blockchains" => db.run(BlockchainsTQ.rows.map(x => (x.name, x.bctype, x.definedBy)).result).map({ list => this._initBCs(list) })
+        case "microservices" => db.run(MicroservicesTQ.rows.map(x => (x.microservice, x.owner)).result).map({ list => this._initMicroservices(list) })
+        case "workloads" => db.run(WorkloadsTQ.rows.map(x => (x.workload, x.owner)).result).map({ list => this._initWorkloads(list) })
       }
     }
 
@@ -181,6 +193,20 @@ object AuthCache {
       }
     }
 
+    /** Put owners of microservices in the cache */
+    def _initMicroservices(credList: Seq[(String,String)]): Unit = {
+      for ((microservice,owner) <- credList) {
+        if (owner != "") _putOwner(microservice, owner)
+      }
+    }
+
+    /** Put owners of workloads in the cache */
+    def _initWorkloads(credList: Seq[(String,String)]): Unit = {
+      for ((workload,owner) <- credList) {
+        if (owner != "") _putOwner(workload, owner)
+      }
+    }
+
     /** Returns Some(Tokens) from the cache for this user/id (but verifies with the db 1st), or None if does not exist */
     def get(id: String): Option[Tokens] = {
       if (Role.isSuperUser(id)) return _get(id)     // root is always initialized from config.json and put in the cache, and should not be changed at runtime
@@ -267,6 +293,8 @@ object AuthCache {
         case "agbots" => AgbotsTQ.getOwner(id).result
         case "bctypes" => BctypesTQ.getOwner(id).result
         case "blockchains" => BlockchainsTQ.getOwner2(id).result
+        case "microservices" => MicroservicesTQ.getOwner(id).result
+        case "workloads" => WorkloadsTQ.getOwner(id).result
       }
       try {
         val ownerVector = Await.result(db.run(a), Duration(3000, MILLISECONDS))
@@ -338,6 +366,8 @@ object AuthCache {
   val agbots = new Cache("agbots")
   val bctypes = new Cache("bctypes")
   val blockchains = new Cache("blockchains")
+  val microservices = new Cache("microservices")
+  val workloads = new Cache("workloads")
 }
 
 /** Authenticates the client credentials and then checks the ACLs for authorization. */
@@ -420,8 +450,21 @@ trait AuthenticationSupport extends ScalatraBase {
             case Access.CREATE => Access.CREATE_BLOCKCHAINS
             case _ => access
           }
+        case TMicroservice(_) => access match {     // a user accessing a microservice
+          case Access.READ => if (iOwnTarget(target)) Access.READ_MY_MICROSERVICES else Access.READ_ALL_MICROSERVICES
+          case Access.WRITE => if (iOwnTarget(target)) Access.WRITE_MY_MICROSERVICES else Access.WRITE_ALL_MICROSERVICES
+          case Access.CREATE => Access.CREATE_MICROSERVICES
+          case _ => access
+        }
+        case TWorkload(_) => access match {     // a user accessing a workload
+          case Access.READ => if (iOwnTarget(target)) Access.READ_MY_WORKLOADS else Access.READ_ALL_WORKLOADS
+          case Access.WRITE => if (iOwnTarget(target)) Access.WRITE_MY_WORKLOADS else Access.WRITE_ALL_WORKLOADS
+          case Access.CREATE => Access.CREATE_WORKLOADS
+          case _ => access
+        }
         case TAction(_) => access      // a user running an action
       }
+      logger.trace("IUser.authorizeTo() access2: "+access2)
       if (Role.hasAuthorization(role, access2)) return this else halt(HttpCode.ACCESS_DENIED, ApiResponse(ApiResponseType.ACCESS_DENIED, accessDeniedMsg(access2)))
     }
 
@@ -435,6 +478,8 @@ trait AuthenticationSupport extends ScalatraBase {
           case TAgbot(id) => AuthCache.agbots.getOwner(id)
           case TBctype(id) => AuthCache.bctypes.getOwner(id)
           case TBlockchain(id) => AuthCache.blockchains.getOwner(id)
+          case TMicroservice(id) => AuthCache.microservices.getOwner(id)
+          case TWorkload(id) => AuthCache.workloads.getOwner(id)
           case _ => return false
         }
         owner match {
@@ -479,6 +524,18 @@ trait AuthenticationSupport extends ScalatraBase {
             case Access.CREATE => Access.CREATE_BLOCKCHAINS
             case _ => access
           }
+        case TMicroservice(_) => access match {     // a device accessing a microservice
+          case Access.READ => Access.READ_ALL_MICROSERVICES
+          case Access.WRITE => Access.WRITE_ALL_MICROSERVICES
+          case Access.CREATE => Access.CREATE_MICROSERVICES
+          case _ => access
+        }
+        case TWorkload(_) => access match {     // a device accessing a workload
+          case Access.READ => Access.READ_ALL_WORKLOADS
+          case Access.WRITE => Access.WRITE_ALL_WORKLOADS
+          case Access.CREATE => Access.CREATE_WORKLOADS
+          case _ => access
+        }
         case TAction(_) => access      // a device running an action
       }
       if (Role.hasAuthorization(Role.DEVICE, access2)) return this else halt(HttpCode.ACCESS_DENIED, ApiResponse(ApiResponseType.ACCESS_DENIED, accessDeniedMsg(access2)))
@@ -519,6 +576,18 @@ trait AuthenticationSupport extends ScalatraBase {
             case Access.CREATE => Access.CREATE_BLOCKCHAINS
             case _ => access
           }
+        case TMicroservice(_) => access match {     // a agbot accessing a microservice
+          case Access.READ => Access.READ_ALL_MICROSERVICES
+          case Access.WRITE => Access.WRITE_ALL_MICROSERVICES
+          case Access.CREATE => Access.CREATE_MICROSERVICES
+          case _ => access
+        }
+        case TWorkload(_) => access match {     // a agbot accessing a workload
+          case Access.READ => Access.READ_ALL_WORKLOADS
+          case Access.WRITE => Access.WRITE_ALL_WORKLOADS
+          case Access.CREATE => Access.CREATE_WORKLOADS
+          case _ => access
+        }
         case TAction(_) => access      // a agbot running an action
       }
       if (Role.hasAuthorization(Role.AGBOT, access2)) return this else halt(HttpCode.ACCESS_DENIED, ApiResponse(ApiResponseType.ACCESS_DENIED, accessDeniedMsg(access2)))
@@ -559,6 +628,18 @@ trait AuthenticationSupport extends ScalatraBase {
             case Access.CREATE => Access.CREATE_BLOCKCHAINS
             case _ => access
           }
+        case TMicroservice(_) => access match {     // a anonymous accessing a microservice
+          case Access.READ => Access.READ_ALL_MICROSERVICES
+          case Access.WRITE => Access.WRITE_ALL_MICROSERVICES
+          case Access.CREATE => Access.CREATE_MICROSERVICES
+          case _ => access
+        }
+        case TWorkload(_) => access match {     // a anonymous accessing a workload
+          case Access.READ => Access.READ_ALL_WORKLOADS
+          case Access.WRITE => Access.WRITE_ALL_WORKLOADS
+          case Access.CREATE => Access.CREATE_WORKLOADS
+          case _ => access
+        }
         case TAction(_) => access      // a anonymous running an action
       }
       if (Role.hasAuthorization(Role.ANONYMOUS, access2)) return this else halt(HttpCode.ACCESS_DENIED, ApiResponse(ApiResponseType.ACCESS_DENIED, accessDeniedMsg(access2)))
@@ -577,6 +658,8 @@ trait AuthenticationSupport extends ScalatraBase {
   case class TAgbot(id: String) extends Target
   case class TBctype(id: String) extends Target      // for bctypes and blockchains only the user that created it can update/delete it
   case class TBlockchain(id: String) extends Target   // this id is a composite of the bc name and bctype
+  case class TMicroservice(id: String) extends Target      // for microservices only the user that created it can update/delete it
+  case class TWorkload(id: String) extends Target      // for workloads only the user that created it can update/delete it
   case class TAction(id: String = "") extends Target    // for post rest api methods that do not target any specific resource (e.g. admin operations)