[verification] Limit the number of concurrent invocations (#485)

Author: igalshilman

scripts/correctness/env.json

@@ -13,6 +13,8 @@
       "RESTATE_METADATA_SERVER__TYPE": "replicated",
       "RESTATE_AUTO_PROVISION": "true",
       "RESTATE_ADVERTISED_ADDRESS": "http://n1:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },
@@ -31,6 +33,8 @@
       "RESTATE_AUTO_PROVISION": "false",
       "RESTATE_METADATA_CLIENT__ADDRESSES": "[http://n1:5122]",
       "RESTATE_ADVERTISED_ADDRESS": "http://n2:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },
@@ -49,6 +53,8 @@
       "RESTATE_AUTO_PROVISION": "false",
       "RESTATE_METADATA_CLIENT__ADDRESSES": "[http://n1:5122]",
       "RESTATE_ADVERTISED_ADDRESS": "http://n3:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },

scripts/perf/env.json

@@ -13,6 +13,8 @@
       "RESTATE_METADATA_SERVER__TYPE": "replicated",
       "RESTATE_AUTO_PROVISION": "true",
       "RESTATE_ADVERTISED_ADDRESS": "http://n1:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },
@@ -31,6 +33,8 @@
       "RESTATE_AUTO_PROVISION": "false",
       "RESTATE_METADATA_CLIENT__ADDRESSES": "[http://n1:5122]",
       "RESTATE_ADVERTISED_ADDRESS": "http://n2:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },
@@ -49,6 +53,8 @@
       "RESTATE_AUTO_PROVISION": "false",
       "RESTATE_METADATA_CLIENT__ADDRESSES": "[http://n1:5122]",
       "RESTATE_ADVERTISED_ADDRESS": "http://n3:5122",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "256",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "1sec",
       "DO_NOT_TRACK": "true"
     }
   },

scripts/snapshotting/env.json

@@ -21,7 +21,9 @@
 			"RESTATE_WORKER__SNAPSHOTS__AWS_SECRET_ACCESS_KEY" : "minioadmin",
 			"RESTATE_WORKER__SNAPSHOTS__DESTINATION" : "s3://restate/foobar",
 			"RESTATE_WORKER__SNAPSHOTS__SNAPSHOT_INTERVAL_NUM_RECORDS" : "10000",
-			"RESTATE_ADMIN__LOG_TRIM_CHECK_INTERVAL" : "10s"
+			"RESTATE_ADMIN__LOG_TRIM_CHECK_INTERVAL" : "10s",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "1024",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "2sec"
     }
   },
   "n2": {
@@ -48,7 +50,9 @@
 			"RESTATE_WORKER__SNAPSHOTS__AWS_ENDPOINT_URL" : "http://minio:9000", 
 			"RESTATE_WORKER__SNAPSHOTS__AWS_ACCESS_KEY_ID" : "minioadmin", 
 			"RESTATE_WORKER__SNAPSHOTS__AWS_SECRET_ACCESS_KEY" : "minioadmin",
-			"RESTATE_WORKER__SNAPSHOTS__DESTINATION" : "s3://restate/foobar"
+			"RESTATE_WORKER__SNAPSHOTS__DESTINATION" : "s3://restate/foobar",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "1024",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "2sec"
 		}
   },
   "n3": {
@@ -78,7 +82,9 @@
 			"RESTATE_WORKER__SNAPSHOTS__AWS_ENDPOINT_URL" : "http://minio:9000", 
 			"RESTATE_WORKER__SNAPSHOTS__AWS_ACCESS_KEY_ID" : "minioadmin", 
 			"RESTATE_WORKER__SNAPSHOTS__AWS_SECRET_ACCESS_KEY" : "minioadmin",
-			"RESTATE_WORKER__SNAPSHOTS__DESTINATION" : "s3://restate/foobar"
+			"RESTATE_WORKER__SNAPSHOTS__DESTINATION" : "s3://restate/foobar",
+			"RESTATE_WORKER__INVOKER__CONCURRENT_INVOCATIONS_LIMIT" : "1024",
+			"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" : "2sec"
 		}
   },
   "interpreter_zero": {