Meta benchmarks

.gitignore

@@ -170,3 +170,16 @@ dmypy.json
 sebs-*
 # cache
 cache
+
+# generated benchmarks
+benchmarks/600.generated
+
+# csv
+*.csv
+
+# generated code by sebs
+scripts/code
+scripts/experiments.json
+
+#
+node_modules/

benchmark_generator/code_composer.py

@@ -0,0 +1,107 @@
+import os
+import uuid
+
+def load_benchmark_code(benchmark_name, language="python"):
+    current_dir = os.getcwd()
+    path_to_code = os.path.join(current_dir, benchmark_name, language, "function.py" if language == "python" else "function.js")
+    if os.path.exists(path_to_code):
+        with open(path_to_code, "r") as source_file:
+            source_code = source_file.read()
+            [_, after_test] = source_code.split("#test")
+            [_, after_import] = after_test.split("#import")
+            [import_part, after_function] = after_import.split("#function")
+            [function_part, run_part] = after_function.split("#run")
+            return {
+                "import": import_part,
+                "function": function_part,
+                "run": run_part
+            }
+    else:
+        print("Path: " + path_to_code + " not exist")
+    return {
+                "import": "",
+                "function": "",
+                "run": ""
+            }
+
+def intend(body):
+    new_body = ""
+    for line in body.splitlines():
+        new_body += "\n\t" + line
+    return new_body
+
+def generate_huge_dict(number_of_elements):
+    return {
+        str(uuid.uuid1()) + "-" + str(i): str(uuid.uuid1()) for i in range(number_of_elements)      # uuid has more predictible size than plain numbers
+    }
+
+def generate_python_handler(config, code_maps):
+    code = "\ndef handler(event):\n"
+    #add invoke of benchmarks
+    handler_function = "result = {}\n"
+    for (number, (benchmark_name, benchmark_config)) in enumerate(config):
+        handler_function += "\nnumber = " + str(number) + "\n"
+        handler_function += "config = " + str(benchmark_config) + "\n"
+        handler_function += code_maps[benchmark_name]["run"]
+
+    if benchmark_name == "artificial_code":
+        number_of_elements = benchmark_config.get("number_of_elements", 0)
+        handler_function += "artificial_dict" + str(number) + " = " + str(generate_huge_dict(number_of_elements))
+
+    handler_function += """\nreturn {'result': result }"""     # dummy result, different doesn't work
+
+    code += intend(handler_function)
+
+    return code
+
+
+def generate_async_nodejs_handler(config, code_maps):
+    code = "\nexports.handler = async function(event) {\n"
+    #add invoke of benchmarks
+    handler_function = """var result = {};\nawait (async () => { return [result, 0] })()"""
+    for (number, (benchmark_name, benchmark_config)) in enumerate(config):
+        handler_function += ".then(async ([result, number]) => {\n"
+        inner_function = "var config = " + str(benchmark_config) + ";\n"
+        inner_function += code_maps[benchmark_name]["run"] + "\n"
+        inner_function += "return [result, number + 1]\n"
+        handler_function += intend(inner_function)
+        handler_function += "\n})\n"
+
+
+    if benchmark_name == "artificial_code":
+        number_of_elements = benchmark_config.get("number_of_elements", 0)
+        handler_function += "var artificial_dict" + str(number) + " = " + str(generate_huge_dict(number_of_elements)) + ";"
+
+    handler_function += """\nreturn {'result': result }\n}"""
+
+    code += intend(handler_function)
+
+    return code
+
+def compose(config, language):
+
+    code = ""
+
+    benchmarks_list = {benchmark for (benchmark, benchmark_config) in config}
+
+    # load code of benchmarks
+    code_maps = {
+        benchmark_name: load_benchmark_code(benchmark_name, language) for benchmark_name in benchmarks_list
+    }
+
+    # add imports
+    for code_map in code_maps.values():
+        code += code_map["import"] + "\n"       # todo not so easy for nodejs- twice imports are not possible
+
+    #add functions
+    for code_map in code_maps.values():
+        code += code_map["function"] + "\n"
+
+    if language == "python":
+        return code + generate_python_handler(config, code_maps)
+    elif language == "async_nodejs":
+        return code + generate_async_nodejs_handler(config, code_maps)
+    else:
+        return ""
+
+

benchmark_generator/disc/async_nodejs/function.js

@@ -0,0 +1,38 @@
+//#test
+var result = {};
+var config = {
+    "block_size": 1024*1024*128
+};
+var number = 0;
+//#import
+var fs = require('fs');
+var uuid = require('uuid');
+var uuidv1 = uuid.v1;
+//#function
+function generate_data_disc(block_size) {
+    return Array(block_size + 1).join('x')
+}
+async function testDisc(block_size) {
+    try {
+        var data = generate_data_disc(entries_number);
+        var path = "/tmp/serverless-benchmark-test-file.json";
+        var t0 = new Date();
+        fs.writeFileSync(path, data);
+        var t1 = new Date();
+        await fs.readFile(path);
+        var t2 = new Date();
+        return { 
+            "write_time" : t1 - t0,
+            "read_time": t2 - t1,
+            "bytes": block_size
+        }
+    } catch (error) {
+        return { "error": error.toString() }
+    }
+};
+//#run
+var block_size = config.block_size;
+await testDisc(block_size).then(returnJson => {
+    result[number] = returnJson;
+  }
+);

benchmark_generator/disc/async_nodejs/package.json

@@ -0,0 +1,13 @@
+{
+    "name": "disc",
+    "version": "1.0.0",
+    "description": "",
+    "author": "",
+    "license": "",
+    "dependencies": {
+      "uuid": "8.2.0"
+    },
+    "devDependencies": {
+    }
+  }
+

benchmark_generator/disc/python/function.py

@@ -0,0 +1,31 @@
+#test
+config = {
+    "block_size": 1024*1024*128
+}
+result = {}
+number = 0
+#import
+import numpy as np
+import time
+import uuid
+import os
+#function
+def test_disc(block_size, file_name):
+    a = np.ones(int(block_size / 4), dtype=np.dtype("int32")) * 2
+    t0 = time.clock()
+    np.save(file_name, a)
+    t1 = time.clock()
+    t2 = time.clock()
+    np.load(file_name)
+    t3 = time.clock()
+
+    write_time = t1 - t0
+    read_time = t3 - t2
+    return {"block_size": block_size,
+        "write_time": write_time,
+        "read_time": read_time}
+#run
+block_size = config.get("block_size", 100)
+file_name = "/tmp/sebs.npy"
+result[str(number)] = test_disc(block_size, file_name)
+print(result)

benchmark_generator/function_input/async_nodejs/function.js

@@ -0,0 +1,22 @@
+//#test
+var config = {
+    "output_size": 100
+};
+var result = {};
+//#import
+var uuid = require('uuid');
+var uuidv1 = uuid.v1;
+//#function
+function fillDict(dictToFill, entries_number) {
+    try {
+        for(var i = 0;i < entries_number;i++) {
+            dictToFill[uuidv1().toString()] = uuidv1().toString()
+        }
+        return dictToFill
+    } catch (error) {
+        return {"Error": error.toString}
+    } 
+}       
+//#run
+var number_of_entries = config["output_size"];
+fillDict(result, number_of_entries);

benchmark_generator/function_input/async_nodejs/package.json

@@ -0,0 +1,13 @@
+{
+    "name": "sleep",
+    "version": "1.0.0",
+    "description": "",
+    "author": "",
+    "license": "",
+    "dependencies": {
+      "uuid": "8.2.0"
+    },
+    "devDependencies": {
+    }
+  }
+

benchmark_generator/function_input/python/function.py

@@ -0,0 +1,16 @@
+#test
+config = {
+    "output_size": 100
+}
+result = {}
+#import
+import uuid
+#function
+def fill_dict(dict_to_fill, number_of_entries):
+    for i in range(number_of_entries):
+        dict_to_fill[str(uuid.uuid1())] = str(uuid.uuid1())
+
+#run
+number_of_entries = config.get("output_size")
+fill_dict(result, number_of_entries)
+print(result)

benchmark_generator/generator.py

@@ -0,0 +1,57 @@
+import sys
+import json
+import code_composer
+import requirements_composer
+import input_composer
+import os
+
+if len(sys.argv) < 2:
+    print("Missing argument, path to config")
+
+with open(sys.argv[1]) as config_file:
+    total_config = json.load(config_file)
+
+if total_config["language"] == "python":
+    config = total_config["config"]
+
+    # Generate directory for benchmark
+    path_to_benchmark = "./../benchmarks/600.generated/620.generated/python"
+    if not os.path.exists(path_to_benchmark):
+        os.makedirs(path_to_benchmark)
+
+    # Push code to benchmarks/600.generated/620.generated/python/function.py
+
+    with open(path_to_benchmark + "/function.py", "w+") as code_file:
+        code = code_composer.compose(config, "python")
+        code_file.write(code)
+
+    # Push requirements to benchmarks/600.generated/620.generated/python/requirements.txt
+    with open(path_to_benchmark + "/requirements.txt", "w+") as requirements_file:
+        requirements = requirements_composer.compose(config)
+        print("Req: " + requirements)
+        requirements_file.write(requirements)
+
+elif total_config["language"] == "async_nodejs":
+    config = total_config["config"]
+
+    # Generate directory for benchmark
+    path_to_benchmark = "./../benchmarks/600.generated/620.generated/nodejs"
+    if not os.path.exists(path_to_benchmark):
+        os.makedirs(path_to_benchmark)
+
+    # Push code to benchmarks/600.generated/620.generated/nodejs/function.js
+
+    with open(path_to_benchmark + "/function.js", "w+") as code_file:
+        code = code_composer.compose(config, "async_nodejs")
+        code_file.write(code)
+
+    # Push requirements to benchmarks/600.generated/620.generated/nodejs/package.json
+    with open(path_to_benchmark + "/package.json", "w+") as requirements_file:
+        requirements = requirements_composer.compose(config, "async_nodejs")
+        print("Req: " + requirements)
+        requirements_file.write(requirements)
+
+# Create input.py file
+with open(path_to_benchmark + "/../input.py", "w+") as input_file:
+    code = input_composer.compose(config)
+    input_file.write(code)

benchmark_generator/input_composer.py

@@ -0,0 +1,37 @@
+import uuid
+
+def compose(config):
+    benchmarks_list = {benchmark for (benchmark, benchmark_config) in config}
+
+    input_dict = {}
+    print(config)
+    for (benchmark, benchmark_config) in config:
+        if benchmark == "function_input" and "input_size" in benchmark_config.keys():
+            # input size is measured by number of elements
+            for i in range(int(benchmark_config["input_size"])):
+                input_dict[str(uuid.uuid1())] = 100
+
+    # add needed values
+
+    # generate code
+    code = ""
+    code += "input_dict = " + str(input_dict) + "\n"
+
+    if "storage" in benchmarks_list:
+        code += """def buckets_count():
+    return (0, 1)\n"""
+    else:
+        code += """def buckets_count():
+    return (0, 0)\n"""
+
+    if "storage" in benchmarks_list:
+        code += """def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+    input_dict = {'bucket': {}}
+    input_dict['bucket']['output'] = output_buckets[0]
+    return input_dict """
+    else:  
+        code += """def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+    return input_dict """
+    return code
+
+

benchmark_generator/memory/async_nodejs/function.js

@@ -0,0 +1,24 @@
+//#test
+var config = {
+  "size_in_bytes": 10485760
+};
+var result = {};
+var number = 0;
+//#import
+var math = require('mathjs');
+//#function
+const testMemory = async (size) => {
+  var t0 = new Date();
+  var a = math.ones([size / 8]);
+  var t1 = new Date();
+  return {
+    "time": t1 - t0,
+    "size_in_bytes": size
+  }
+};
+//#run
+var array_size_in_bytes = config["size_in_bytes"];
+await testMemory(array_size_in_bytes).then(returnJson => {
+    result[number] = returnJson;
+  }
+);