From 7f4ce43ad0862fbd63229c37be577339d35997fb Mon Sep 17 00:00:00 2001 From: eliyahu77 Date: Tue, 23 Aug 2022 17:23:29 +0300 Subject: [PATCH] add copyright Signed-off-by: eliyahu77 --- CSV_to_Parquet/csv_to_parquet.py | 14 + copy_kv_table/copy_kv_table.py | 1004 +++++++++-------- create_schema/create_schema.py | 14 + create_schema/igz_nosql_web.py | 14 + flight_streaming/cleanup_stream.sh | 14 + flight_streaming/create_stream.sh | 14 + flight_streaming/python/consume.py | 14 + flight_streaming/python/loader.py | 14 + .../src/main/java/com/iguazio/bo/Car.java | 19 + .../drivers/IguazioKvReaderDriver.java | 19 + .../drivers/IguazioStreamConsumerDriver.java | 19 + .../drivers/KafkaStreamProducerDriver.java | 19 + .../KafkaToIguazioStreamIngestionDriver.java | 19 + .../drivers/KafkaToKvIngestionDriver.java | 19 + .../java/com/iguazio/function/CarDecoder.java | 19 + .../function/IguazioStreamVoidFunction.java | 19 + .../function/KVIngestionVoidFunction.java | 19 + .../iguazio/function/StreamCarFunction.java | 19 + .../function/StreamIngestionVoidFunction.java | 19 + .../iguazio/function/StreamVoidFunction.java | 19 + netops_demo/Makefile | 14 + .../src/github.com/v3io/demos/Dockerfile | 14 + .../demos/functions/ingest/anodot/appender.go | 19 + .../v3io/demos/functions/ingest/ingest.go | 19 + .../demos/functions/ingest/ingest_test.go | 19 + .../golang/src/github.com/v3io/demos/main.go | 19 + netops_demo/py/Dockerfile | 14 + netops_demo/py/functions/generate/generate.py | 14 + .../py/functions/generate/generate_test.py | 14 + netops_demo/py/libs/generator/__init__.py | 14 + .../py/libs/generator/baseline/__init__.py | 14 + .../py/libs/generator/baseline/company.py | 14 + .../py/libs/generator/baseline/location.py | 14 + .../py/libs/generator/baseline/normal.py | 14 + netops_demo/py/libs/generator/deployment.py | 14 + netops_demo/py/libs/generator/device.py | 14 + netops_demo/py/libs/generator/metric.py | 14 + netops_demo/py/libs/nuclio_sdk/exceptions.py | 14 + netops_demo/py/run_generator.py | 14 + taxi_nosql/Makefile | 14 + taxi_nosql/create_random_driver_data.py | 14 + taxi_nosql/create_random_driver_data.sh | 14 + taxi_nosql/create_random_passenger_data.py | 14 + taxi_nosql/create_random_passenger_data.sh | 14 + taxi_nosql/igz_nosql_web.py | 14 + taxi_nosql/load_driver_data.py | 14 + taxi_nosql/load_driver_data.sh | 14 + taxi_nosql/load_passenger_data.py | 14 + taxi_nosql/load_passenger_data.sh | 14 + taxi_nosql/nuclio_golang/DriverKV.go | 19 + taxi_nosql/nuclio_golang/function.yaml | 14 + .../nuclio_golang/insert_driver_nuclio.py | 96 +- taxi_nosql/nuclio_python/Dockerfile | 14 + .../functions/generate_data/generate_data.py | 250 ++-- .../nuclio_python/functions/ingest/ingest.py | 14 + taxi_nosql/nuclio_python/generate_data.yaml | 14 + taxi_nosql/nuclio_python/ingest.yaml | 14 + taxi_nosql/remove_all_data.sh | 14 + taxi_nosql/set_env.sh | 14 + taxi_nosql/try.py | 14 + taxi_streaming/consume_drivers_stream_data.py | 14 + taxi_streaming/create_drivers_stream.sh | 14 + taxi_streaming/create_random_drivers_data.py | 14 + taxi_streaming/create_random_drivers_data.sh | 14 + taxi_streaming/delete_stream.sh | 14 + taxi_streaming/spark-submit_cmd.sh | 14 + taxi_streaming/stream_drivers_data.py | 14 + taxi_streaming/stream_drivers_data.sh | 14 + 68 files changed, 1691 insertions(+), 654 deletions(-) diff --git a/CSV_to_Parquet/csv_to_parquet.py b/CSV_to_Parquet/csv_to_parquet.py index 64b8a06..612342b 100644 --- a/CSV_to_Parquet/csv_to_parquet.py +++ b/CSV_to_Parquet/csv_to_parquet.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from pyspark.sql import SparkSession from pyspark.sql.types import * import datetime diff --git a/copy_kv_table/copy_kv_table.py b/copy_kv_table/copy_kv_table.py index 970f48e..9809749 100755 --- a/copy_kv_table/copy_kv_table.py +++ b/copy_kv_table/copy_kv_table.py @@ -1,495 +1,509 @@ -#!/usr/bin/python - -import requests -import argparse -import json -import Queue -import threading -import time -import logging - -INFO_PLUS_LOG_LEVEL = 15 -INFO_PLUS_LEVEL_NAME = "INFO_PLUS" - - -def add_logging_level(level_name, level_num, method_name=None): - """ - Comprehensively adds a new logging level to the `logging` module and the - currently configured logging class. - - `levelName` becomes an attribute of the `logging` module with the value - `levelNum`. `methodName` becomes a convenience method for both `logging` - itself and the class returned by `logging.getLoggerClass()` (usually just - `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is - used. - - To avoid accidental clobberings of existing attributes, this method will - raise an `AttributeError` if the level name is already an attribute of the - `logging` module or if the method name is already present - """ - - if not method_name: - method_name = level_name.lower() - - if hasattr(logging, level_name): - raise AttributeError('{} already defined in logging module'.format(level_name)) - if hasattr(logging, method_name): - raise AttributeError('{} already defined in logging module'.format(method_name)) - if hasattr(logging.getLoggerClass(), method_name): - raise AttributeError('{} already defined in logger class'.format(method_name)) - - # This method was inspired by the answers to Stack Overflow post - # http://stackoverflow.com/q/2183233/2988730, especially - # http://stackoverflow.com/a/13638084/2988730 - def log_for_level(self, message, *args, **kwargs): - if self.isEnabledFor(level_num): - self._log(level_num, message, args, **kwargs) - - def log_to_root(message, *args, **kwargs): - logging.log(level_num, message, *args, **kwargs) - - logging.addLevelName(level_num, level_name) - setattr(logging, level_name, level_num) - setattr(logging.getLoggerClass(), method_name, log_for_level) - setattr(logging, method_name, log_to_root) - - -def parse_arguments(): - - DEF_HTTP_PORT = 8081 - DEF_HTTPS_PORT = 8443 - LOCALHOST = "127.0.0.1" - - parser = argparse.ArgumentParser( - description="Copy onve KV table to another location, on the same system/container," - " or a different one. Only copies attributes, not actual files") - - parser.add_argument("-i", "--source-ip", - type=str, - default=LOCALHOST, - required=False, - help="IP address of the web-gateway service on the source system. Default = 'localhost'.") - parser.add_argument("-c", "--source-container", - type=str, - required=True, - help="The name of the source table's parent container.") - parser.add_argument("-t", "--source-table-path", - type=str, - required=True, - help="Path to the source table's root directory within the container.") - parser.add_argument("-g", "--source_segments", - type=int, - default=36, - help="The number of segments to use in the source table-items scan. A value higher" - " than 1 configures a parallel multi-segment scan. Default = 36.") - parser.add_argument("-d", "--destination-ip", - type=str, - required=False, - help="IP address of the web-gateway service on the destination system." - " Default = same as source.") - parser.add_argument("-x", "--destination-container", - type=str, - required=False, - help="The name of the destination table's parent container.") - parser.add_argument("-y", "--destination-table-path", - type=str, - required=True, - help="Path to the destination table's root directory within the container.") - parser.add_argument("-z", "--destination_parallelism", - type=int, - default=72, - help="The number of writers to use for writing to the destination. Default = 72.") - parser.add_argument("-p", "--port", - type=int, - required=False, - help="TCP port of the web-gateway services on both source and destination. Must be the same." - " Default = 8081 for HTTP and 8443 for HTTPS (see the -s|--secure option).") - parser.add_argument("-s", "--secure", - action="store_true", - required=False, - help="Use HTTPS (without a certificate verification) instead of HTTP, for both" - " source and destination.") - parser.add_argument("-u", "--user", - type=str, - required=False, - help="Username to be used for HTTP authentication together with the password set" - " with the -w or --password option, for both source and destination.") - parser.add_argument("-w", "--password", - type=str, - required=False, - help="Password to be used for HTTP authentication together with the username set with" - " the -u or --user option, for both source and destination.") - parser.add_argument("-v", "--verbose", - action="count", - default=0, - help="Increase the verbosity level of the command-line output.") - - args = parser.parse_args() - # Custom parameter handling - if args.destination_ip is None: - args.destination_ip = args.source_ip - - if args.destination_container is None: - args.destination_container = args.source_container - - if args.port is None: # If port isn't specified, assign port defaults based on the value of "secure" - if args.secure: - args.port = DEF_HTTPS_PORT - else: - args.port = DEF_HTTP_PORT - - if (args.user is None and args.password is not None) or (args.user is not None and args.password is None): - parser.error("User and password must both be provided if one is provided.") - - return args - - -class CopyKvController(threading.Thread): - - def __init__(self): - threading.Thread.__init__(self) - self._writers = 0 - self._readers = 0 - self._readers_launched = False - self._writers_launched = False - - def print_status(self): - logging.info_plus("Copy KV Controller - status - {} readers, {} writers".format(self._readers, self._writers)) - - def register_reader(self): - self._readers = self._readers + 1 - self._readers_launched = True - - def de_register_reader(self): - self._readers = self._readers - 1 - - def register_writer(self): - self._writers = self._writers + 1 - self._writers_launched = True - - def de_register_writer(self): - self._writers = self._writers - 1 - - def readers_done(self): - return self._readers == 0 - - def writers_done(self): - return self._writers == 0 - - def readers_launched(self): - return self._readers_launched - - def work_finished(self): - if self._readers_launched and self._writers_launched and self.readers_done() and self.writers_done(): - return True - else: - return False - - -class ItemsQueue: - - def __init__(self): - self.__items_queue = Queue.Queue() - self.__items_inserted = 0 - self.__items_removed = 0 - - def put(self, item): - self.__items_inserted = self.__items_inserted + 1 - self.__items_queue.put(item) - - def get(self): - if self.__items_queue.empty(): - return - else: - self.__items_removed = self.__items_removed + 1 - return self.__items_queue.get() - - def insert_count(self): - return self.__items_inserted - - def removed_count(self): - return self.__items_removed - - -class ItemsQueueMonitor(threading.Thread): - - def __init__(self, - items_queue, - verbosity, - kv_cont): - threading.Thread.__init__(self) - self.items_queue = items_queue - self.verbosity = verbosity - self.kv_cont = kv_cont - - def run(self): - while not self.kv_cont.work_finished(): - logging.info_plus("Queue monitor - read = {}, written = {}".format(self.items_queue.insert_count(), - self.items_queue.removed_count())) - self.kv_cont.print_status() - time.sleep(5) - logging.info_plus("Queue monitor is done... read = {}, written = {}".format(self.items_queue.insert_count(), - self.items_queue.removed_count())) - - -class Reader(threading.Thread): - - def __init__(self, - s, - url, - slice_num, - total_slices, - items_queue, - kv_cont, - verbosity=0): - threading.Thread.__init__(self) - self.s = s - self.url = url - self.slice = slice_num - self.total_slices = total_slices - self.items_queue = items_queue - self.kv_cont = kv_cont - self.verbosity = verbosity - - self.kv_cont.register_reader() - - def run(self): - - read_request = dict() - read_request["AttributesToGet"] = "*" - read_request["Segment"] = self.slice - read_request["TotalSegment"] = self.total_slices - - # Connection : keep-alive will tell requests to use connection pooling, which is good. - read_headers = dict() - read_headers["X-v3io-function"] = "GetItems" - read_headers["Connection"] = "keep-alive" - - total_items = 0 - - error_encountered = False - - while not error_encountered: - # marker will get updated if needed - read_request_json = json.dumps(read_request) - - read_json_result = {} - retries_left = 3 - while True: # will break upon success - try: - read_json_result = self.s.put(self.url, data=read_request_json, headers=read_headers, timeout=3.0) - break - except requests.Timeout as e: - logging.info_plus("Reader timed out, {} retries left: {}".format(retries_left, e)) - retries_left = retries_left - 1 - if retries_left == 0: - logging.critical("Exiting Reader - maximum number of retries exceeded") - error_encountered = True - break - else: - time.sleep(6 - retries_left) - except requests.ConnectionError as e: - logging.info("Error establishing connection: " + str(e)) - logging.info("Request: url {}; data {}; headers {};".format(self.url, read_request_json, read_headers)) - logging.info("Returned: read_json_result {};".format(read_json_result)) - error_encountered = True - break - except Exception as e: - logging.info("Error establishing connection: " + str(e)) - logging.info("Request: url {}; data {}; headers {};".format(self.url, read_request_json, read_headers)) - logging.info("Returned: read_json_result {};".format(read_json_result)) - error_encountered = True - break - - if error_encountered or read_json_result is None or read_json_result.status_code != requests.codes.ok: - logging.critical("Error while reading items") - break - else: - # push items into queue - read_result = json.loads(read_json_result.content) - num_of_items = read_result["NumItems"] - - for i in range(0, num_of_items): - self.items_queue.put(read_result["Items"][i]) - total_items = total_items + 1 - - # prepare next read request, or finish - if read_result["LastItemIncluded"] == "FALSE": - read_request["Marker"] = read_result["NextMarker"] - else: - logging.info_plus("Reader #{} completed, {} records processed".format(self.slice, total_items)) - break - self.kv_cont.de_register_reader() - -class Writer(threading.Thread): - - def __init__(self, - s, - base_url, - instance, # mainly for debug - items_queue, - kv_cont, - verbosity=0): - threading.Thread.__init__(self) - self.s = s - self.base_url = base_url - self.instance = instance - self.items_queue = items_queue - self.kv_cont = kv_cont - self.verbosity = verbosity - - self.kv_cont.register_writer() - - def run(self): - - # Connection : keep-alive will tell requests to use connection pooling, which is good. - write_headers = dict() - write_headers["X-v3io-function"] = "PutItem" - write_headers["Connection"] = "keep-alive" - - startup_retries = 3 - while not self.kv_cont.readers_launched(): - startup_retries = startup_retries - 1 - if startup_retries == 0: - self.kv_cont.de_register_writer() - logging.debug("Writer quit due to initail lack of data...") - return - time.sleep(2.0) - - items_written = 0 - error_encountered = False - while not error_encountered: # While there are items to write - - curr_item = self.items_queue.get() - - # if queue is empty, need to figure out if work is completed, or need to wait for more - if curr_item is None: - if self.kv_cont.readers_done(): - logging.info_plus("Writer {} is done, {} items written".format(self.instance, items_written)) - break - else: - logging.debug("Writer {} has no data to work on, but readers are not done..." - " sleeping".format(self.instance)) - time.sleep(5.0) - - else: - # Convert record to writeable form - url = self.base_url + curr_item["__name"]["S"] - item = dict() - item["Item"] = curr_item - item["Item"].pop("__name") - json_item = json.dumps(item) - - write_json_result = {} - retries_left = 7 - while True: # for retries - try: - write_json_result = self.s.put(url, data=json_item, headers=write_headers, timeout=5.0) - items_written = items_written + 1 - break - # TODO: Missing de_register handling on errors. - except requests.Timeout as e: - logging.info_plus("Writer timed out, {} retries left: {}".format(retries_left, e)) - retries_left = retries_left - 1 - if retries_left == 0: - logging.critical("Exiting Writer - maximum number of retries exceeded") - error_encountered = True - break - else: - time.sleep(20 - retries_left) - except requests.ConnectionError as e: - logging.info("Error establishing connection: " + str(e)) - logging.info("Request: url {}; data {}; headers {};".format(url, json_item, write_headers)) - logging.info("Returned: write_json_result {};".format(write_json_result)) - error_encountered = True - break - except Exception as e: - logging.info("Error establishing connection: " + str(e)) - logging.info("Request: url {}; data {}; headers {};".format(url, json_item, write_headers)) - logging.info("Returned: write_json_result {};".format(write_json_result)) - error_encountered = True - break - - if error_encountered or write_json_result is None or write_json_result.status_code != requests.codes.no_content: - logging.critical("Error while writing items") - break # from processing loop - - # orderly exit - self.kv_cont.de_register_writer() - - -def main(): - args = parse_arguments() - - add_logging_level(INFO_PLUS_LEVEL_NAME, INFO_PLUS_LOG_LEVEL) - - if args.verbose >= 2: - logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.DEBUG) - elif args.verbose == 1: - logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.INFO_PLUS) - else: - logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.INFO) - - # Suppress other modules: - logging.getLogger("requests").setLevel(logging.ERROR) - logging.getLogger("urllib3").setLevel(logging.ERROR) - - logging.debug("Program arguments after parsing and processing:") - logging.debug(args) - - protocol = "https" if args.secure else "http" - - source_url = protocol + "://" + args.source_ip + ":" + str(args.port) + "/" \ - + args.source_container + "/" + args.source_table_path + "/" - - destination_url = protocol + "://" + args.destination_ip + ":" + str(args.port) + "/" \ - + args.destination_container + "/" + args.destination_table_path + "/" - - logging.info_plus("Source: {}; Destination: {}".format(source_url, destination_url)) - - ss = requests.Session() - sd = requests.Session() - if args.user is not None and args.password is not None: - ss.auth = (args.user, args.password) - sd.auth = (args.user, args.password) - - if args.secure: - ss.verify = False - sd.verify = False - requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) - - kv_controller = CopyKvController() - - items_queue = ItemsQueue() - items_queue_monitor = ItemsQueueMonitor(items_queue, args.verbose, kv_controller) - items_queue_monitor.start() - - readers = [{} for _ in range(args.source_segments)] - for i in range(0, args.source_segments): - readers[i] = Reader(ss, - source_url, - slice_num=i, - total_slices=args.source_segments, - items_queue=items_queue, - kv_cont=kv_controller, - verbosity=args.verbose) - readers[i].start() - - writers = [{} for _ in range(args.destination_parallelism)] - for i in range(0, args.destination_parallelism): - writers[i] = Writer(sd, - base_url=destination_url, - instance=i, - items_queue=items_queue, - kv_cont=kv_controller, - verbosity=args.verbose) - writers[i].start() - - -main() +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#!/usr/bin/python + +import requests +import argparse +import json +import Queue +import threading +import time +import logging + +INFO_PLUS_LOG_LEVEL = 15 +INFO_PLUS_LEVEL_NAME = "INFO_PLUS" + + +def add_logging_level(level_name, level_num, method_name=None): + """ + Comprehensively adds a new logging level to the `logging` module and the + currently configured logging class. + + `levelName` becomes an attribute of the `logging` module with the value + `levelNum`. `methodName` becomes a convenience method for both `logging` + itself and the class returned by `logging.getLoggerClass()` (usually just + `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is + used. + + To avoid accidental clobberings of existing attributes, this method will + raise an `AttributeError` if the level name is already an attribute of the + `logging` module or if the method name is already present + """ + + if not method_name: + method_name = level_name.lower() + + if hasattr(logging, level_name): + raise AttributeError('{} already defined in logging module'.format(level_name)) + if hasattr(logging, method_name): + raise AttributeError('{} already defined in logging module'.format(method_name)) + if hasattr(logging.getLoggerClass(), method_name): + raise AttributeError('{} already defined in logger class'.format(method_name)) + + # This method was inspired by the answers to Stack Overflow post + # http://stackoverflow.com/q/2183233/2988730, especially + # http://stackoverflow.com/a/13638084/2988730 + def log_for_level(self, message, *args, **kwargs): + if self.isEnabledFor(level_num): + self._log(level_num, message, args, **kwargs) + + def log_to_root(message, *args, **kwargs): + logging.log(level_num, message, *args, **kwargs) + + logging.addLevelName(level_num, level_name) + setattr(logging, level_name, level_num) + setattr(logging.getLoggerClass(), method_name, log_for_level) + setattr(logging, method_name, log_to_root) + + +def parse_arguments(): + + DEF_HTTP_PORT = 8081 + DEF_HTTPS_PORT = 8443 + LOCALHOST = "127.0.0.1" + + parser = argparse.ArgumentParser( + description="Copy onve KV table to another location, on the same system/container," + " or a different one. Only copies attributes, not actual files") + + parser.add_argument("-i", "--source-ip", + type=str, + default=LOCALHOST, + required=False, + help="IP address of the web-gateway service on the source system. Default = 'localhost'.") + parser.add_argument("-c", "--source-container", + type=str, + required=True, + help="The name of the source table's parent container.") + parser.add_argument("-t", "--source-table-path", + type=str, + required=True, + help="Path to the source table's root directory within the container.") + parser.add_argument("-g", "--source_segments", + type=int, + default=36, + help="The number of segments to use in the source table-items scan. A value higher" + " than 1 configures a parallel multi-segment scan. Default = 36.") + parser.add_argument("-d", "--destination-ip", + type=str, + required=False, + help="IP address of the web-gateway service on the destination system." + " Default = same as source.") + parser.add_argument("-x", "--destination-container", + type=str, + required=False, + help="The name of the destination table's parent container.") + parser.add_argument("-y", "--destination-table-path", + type=str, + required=True, + help="Path to the destination table's root directory within the container.") + parser.add_argument("-z", "--destination_parallelism", + type=int, + default=72, + help="The number of writers to use for writing to the destination. Default = 72.") + parser.add_argument("-p", "--port", + type=int, + required=False, + help="TCP port of the web-gateway services on both source and destination. Must be the same." + " Default = 8081 for HTTP and 8443 for HTTPS (see the -s|--secure option).") + parser.add_argument("-s", "--secure", + action="store_true", + required=False, + help="Use HTTPS (without a certificate verification) instead of HTTP, for both" + " source and destination.") + parser.add_argument("-u", "--user", + type=str, + required=False, + help="Username to be used for HTTP authentication together with the password set" + " with the -w or --password option, for both source and destination.") + parser.add_argument("-w", "--password", + type=str, + required=False, + help="Password to be used for HTTP authentication together with the username set with" + " the -u or --user option, for both source and destination.") + parser.add_argument("-v", "--verbose", + action="count", + default=0, + help="Increase the verbosity level of the command-line output.") + + args = parser.parse_args() + # Custom parameter handling + if args.destination_ip is None: + args.destination_ip = args.source_ip + + if args.destination_container is None: + args.destination_container = args.source_container + + if args.port is None: # If port isn't specified, assign port defaults based on the value of "secure" + if args.secure: + args.port = DEF_HTTPS_PORT + else: + args.port = DEF_HTTP_PORT + + if (args.user is None and args.password is not None) or (args.user is not None and args.password is None): + parser.error("User and password must both be provided if one is provided.") + + return args + + +class CopyKvController(threading.Thread): + + def __init__(self): + threading.Thread.__init__(self) + self._writers = 0 + self._readers = 0 + self._readers_launched = False + self._writers_launched = False + + def print_status(self): + logging.info_plus("Copy KV Controller - status - {} readers, {} writers".format(self._readers, self._writers)) + + def register_reader(self): + self._readers = self._readers + 1 + self._readers_launched = True + + def de_register_reader(self): + self._readers = self._readers - 1 + + def register_writer(self): + self._writers = self._writers + 1 + self._writers_launched = True + + def de_register_writer(self): + self._writers = self._writers - 1 + + def readers_done(self): + return self._readers == 0 + + def writers_done(self): + return self._writers == 0 + + def readers_launched(self): + return self._readers_launched + + def work_finished(self): + if self._readers_launched and self._writers_launched and self.readers_done() and self.writers_done(): + return True + else: + return False + + +class ItemsQueue: + + def __init__(self): + self.__items_queue = Queue.Queue() + self.__items_inserted = 0 + self.__items_removed = 0 + + def put(self, item): + self.__items_inserted = self.__items_inserted + 1 + self.__items_queue.put(item) + + def get(self): + if self.__items_queue.empty(): + return + else: + self.__items_removed = self.__items_removed + 1 + return self.__items_queue.get() + + def insert_count(self): + return self.__items_inserted + + def removed_count(self): + return self.__items_removed + + +class ItemsQueueMonitor(threading.Thread): + + def __init__(self, + items_queue, + verbosity, + kv_cont): + threading.Thread.__init__(self) + self.items_queue = items_queue + self.verbosity = verbosity + self.kv_cont = kv_cont + + def run(self): + while not self.kv_cont.work_finished(): + logging.info_plus("Queue monitor - read = {}, written = {}".format(self.items_queue.insert_count(), + self.items_queue.removed_count())) + self.kv_cont.print_status() + time.sleep(5) + logging.info_plus("Queue monitor is done... read = {}, written = {}".format(self.items_queue.insert_count(), + self.items_queue.removed_count())) + + +class Reader(threading.Thread): + + def __init__(self, + s, + url, + slice_num, + total_slices, + items_queue, + kv_cont, + verbosity=0): + threading.Thread.__init__(self) + self.s = s + self.url = url + self.slice = slice_num + self.total_slices = total_slices + self.items_queue = items_queue + self.kv_cont = kv_cont + self.verbosity = verbosity + + self.kv_cont.register_reader() + + def run(self): + + read_request = dict() + read_request["AttributesToGet"] = "*" + read_request["Segment"] = self.slice + read_request["TotalSegment"] = self.total_slices + + # Connection : keep-alive will tell requests to use connection pooling, which is good. + read_headers = dict() + read_headers["X-v3io-function"] = "GetItems" + read_headers["Connection"] = "keep-alive" + + total_items = 0 + + error_encountered = False + + while not error_encountered: + # marker will get updated if needed + read_request_json = json.dumps(read_request) + + read_json_result = {} + retries_left = 3 + while True: # will break upon success + try: + read_json_result = self.s.put(self.url, data=read_request_json, headers=read_headers, timeout=3.0) + break + except requests.Timeout as e: + logging.info_plus("Reader timed out, {} retries left: {}".format(retries_left, e)) + retries_left = retries_left - 1 + if retries_left == 0: + logging.critical("Exiting Reader - maximum number of retries exceeded") + error_encountered = True + break + else: + time.sleep(6 - retries_left) + except requests.ConnectionError as e: + logging.info("Error establishing connection: " + str(e)) + logging.info("Request: url {}; data {}; headers {};".format(self.url, read_request_json, read_headers)) + logging.info("Returned: read_json_result {};".format(read_json_result)) + error_encountered = True + break + except Exception as e: + logging.info("Error establishing connection: " + str(e)) + logging.info("Request: url {}; data {}; headers {};".format(self.url, read_request_json, read_headers)) + logging.info("Returned: read_json_result {};".format(read_json_result)) + error_encountered = True + break + + if error_encountered or read_json_result is None or read_json_result.status_code != requests.codes.ok: + logging.critical("Error while reading items") + break + else: + # push items into queue + read_result = json.loads(read_json_result.content) + num_of_items = read_result["NumItems"] + + for i in range(0, num_of_items): + self.items_queue.put(read_result["Items"][i]) + total_items = total_items + 1 + + # prepare next read request, or finish + if read_result["LastItemIncluded"] == "FALSE": + read_request["Marker"] = read_result["NextMarker"] + else: + logging.info_plus("Reader #{} completed, {} records processed".format(self.slice, total_items)) + break + self.kv_cont.de_register_reader() + +class Writer(threading.Thread): + + def __init__(self, + s, + base_url, + instance, # mainly for debug + items_queue, + kv_cont, + verbosity=0): + threading.Thread.__init__(self) + self.s = s + self.base_url = base_url + self.instance = instance + self.items_queue = items_queue + self.kv_cont = kv_cont + self.verbosity = verbosity + + self.kv_cont.register_writer() + + def run(self): + + # Connection : keep-alive will tell requests to use connection pooling, which is good. + write_headers = dict() + write_headers["X-v3io-function"] = "PutItem" + write_headers["Connection"] = "keep-alive" + + startup_retries = 3 + while not self.kv_cont.readers_launched(): + startup_retries = startup_retries - 1 + if startup_retries == 0: + self.kv_cont.de_register_writer() + logging.debug("Writer quit due to initail lack of data...") + return + time.sleep(2.0) + + items_written = 0 + error_encountered = False + while not error_encountered: # While there are items to write + + curr_item = self.items_queue.get() + + # if queue is empty, need to figure out if work is completed, or need to wait for more + if curr_item is None: + if self.kv_cont.readers_done(): + logging.info_plus("Writer {} is done, {} items written".format(self.instance, items_written)) + break + else: + logging.debug("Writer {} has no data to work on, but readers are not done..." + " sleeping".format(self.instance)) + time.sleep(5.0) + + else: + # Convert record to writeable form + url = self.base_url + curr_item["__name"]["S"] + item = dict() + item["Item"] = curr_item + item["Item"].pop("__name") + json_item = json.dumps(item) + + write_json_result = {} + retries_left = 7 + while True: # for retries + try: + write_json_result = self.s.put(url, data=json_item, headers=write_headers, timeout=5.0) + items_written = items_written + 1 + break + # TODO: Missing de_register handling on errors. + except requests.Timeout as e: + logging.info_plus("Writer timed out, {} retries left: {}".format(retries_left, e)) + retries_left = retries_left - 1 + if retries_left == 0: + logging.critical("Exiting Writer - maximum number of retries exceeded") + error_encountered = True + break + else: + time.sleep(20 - retries_left) + except requests.ConnectionError as e: + logging.info("Error establishing connection: " + str(e)) + logging.info("Request: url {}; data {}; headers {};".format(url, json_item, write_headers)) + logging.info("Returned: write_json_result {};".format(write_json_result)) + error_encountered = True + break + except Exception as e: + logging.info("Error establishing connection: " + str(e)) + logging.info("Request: url {}; data {}; headers {};".format(url, json_item, write_headers)) + logging.info("Returned: write_json_result {};".format(write_json_result)) + error_encountered = True + break + + if error_encountered or write_json_result is None or write_json_result.status_code != requests.codes.no_content: + logging.critical("Error while writing items") + break # from processing loop + + # orderly exit + self.kv_cont.de_register_writer() + + +def main(): + args = parse_arguments() + + add_logging_level(INFO_PLUS_LEVEL_NAME, INFO_PLUS_LOG_LEVEL) + + if args.verbose >= 2: + logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.DEBUG) + elif args.verbose == 1: + logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.INFO_PLUS) + else: + logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.INFO) + + # Suppress other modules: + logging.getLogger("requests").setLevel(logging.ERROR) + logging.getLogger("urllib3").setLevel(logging.ERROR) + + logging.debug("Program arguments after parsing and processing:") + logging.debug(args) + + protocol = "https" if args.secure else "http" + + source_url = protocol + "://" + args.source_ip + ":" + str(args.port) + "/" \ + + args.source_container + "/" + args.source_table_path + "/" + + destination_url = protocol + "://" + args.destination_ip + ":" + str(args.port) + "/" \ + + args.destination_container + "/" + args.destination_table_path + "/" + + logging.info_plus("Source: {}; Destination: {}".format(source_url, destination_url)) + + ss = requests.Session() + sd = requests.Session() + if args.user is not None and args.password is not None: + ss.auth = (args.user, args.password) + sd.auth = (args.user, args.password) + + if args.secure: + ss.verify = False + sd.verify = False + requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) + + kv_controller = CopyKvController() + + items_queue = ItemsQueue() + items_queue_monitor = ItemsQueueMonitor(items_queue, args.verbose, kv_controller) + items_queue_monitor.start() + + readers = [{} for _ in range(args.source_segments)] + for i in range(0, args.source_segments): + readers[i] = Reader(ss, + source_url, + slice_num=i, + total_slices=args.source_segments, + items_queue=items_queue, + kv_cont=kv_controller, + verbosity=args.verbose) + readers[i].start() + + writers = [{} for _ in range(args.destination_parallelism)] + for i in range(0, args.destination_parallelism): + writers[i] = Writer(sd, + base_url=destination_url, + instance=i, + items_queue=items_queue, + kv_cont=kv_controller, + verbosity=args.verbose) + writers[i].start() + + +main() diff --git a/create_schema/create_schema.py b/create_schema/create_schema.py index 8b57659..2e98c40 100644 --- a/create_schema/create_schema.py +++ b/create_schema/create_schema.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/python import requests diff --git a/create_schema/igz_nosql_web.py b/create_schema/igz_nosql_web.py index a9fe338..c44ecc5 100644 --- a/create_schema/igz_nosql_web.py +++ b/create_schema/igz_nosql_web.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import requests import json import threading diff --git a/flight_streaming/cleanup_stream.sh b/flight_streaming/cleanup_stream.sh index 2bdc086..2e5fa3a 100755 --- a/flight_streaming/cleanup_stream.sh +++ b/flight_streaming/cleanup_stream.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash NGINX_IP='10.90.1.101' diff --git a/flight_streaming/create_stream.sh b/flight_streaming/create_stream.sh index 29fcab0..ccfbda6 100755 --- a/flight_streaming/create_stream.sh +++ b/flight_streaming/create_stream.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash export NGINX_IP='10.90.1.101' diff --git a/flight_streaming/python/consume.py b/flight_streaming/python/consume.py index ab67fb1..007a871 100644 --- a/flight_streaming/python/consume.py +++ b/flight_streaming/python/consume.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from pyspark.sql import SparkSession from pyspark.streaming import StreamingContext from v3io.spark.streaming import * diff --git a/flight_streaming/python/loader.py b/flight_streaming/python/loader.py index 14cabff..7759269 100644 --- a/flight_streaming/python/loader.py +++ b/flight_streaming/python/loader.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import base64 import collections import json diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/bo/Car.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/bo/Car.java index efbc376..5de9b33 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/bo/Car.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/bo/Car.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.bo; public class Car { diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioKvReaderDriver.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioKvReaderDriver.java index fd01371..532a819 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioKvReaderDriver.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioKvReaderDriver.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.drivers; import org.apache.spark.sql.Dataset; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioStreamConsumerDriver.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioStreamConsumerDriver.java index ec84dda..31a68e0 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioStreamConsumerDriver.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/IguazioStreamConsumerDriver.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.drivers; import java.io.IOException; import java.io.Serializable; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaStreamProducerDriver.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaStreamProducerDriver.java index a1af507..c986ee1 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaStreamProducerDriver.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaStreamProducerDriver.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.drivers; import java.util.Properties; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToIguazioStreamIngestionDriver.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToIguazioStreamIngestionDriver.java index ecedbdf..f98d893 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToIguazioStreamIngestionDriver.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToIguazioStreamIngestionDriver.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.drivers; import java.io.IOException; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToKvIngestionDriver.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToKvIngestionDriver.java index 43b85bf..1de2078 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToKvIngestionDriver.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/drivers/KafkaToKvIngestionDriver.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.drivers; import java.io.Serializable; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/CarDecoder.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/CarDecoder.java index cf6ad4f..e826f5b 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/CarDecoder.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/CarDecoder.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import java.nio.charset.Charset; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/IguazioStreamVoidFunction.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/IguazioStreamVoidFunction.java index 8e7ed9f..06adfbe 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/IguazioStreamVoidFunction.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/IguazioStreamVoidFunction.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import org.apache.spark.api.java.JavaRDD; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/KVIngestionVoidFunction.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/KVIngestionVoidFunction.java index b2f20ba..fd9774b 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/KVIngestionVoidFunction.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/KVIngestionVoidFunction.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import org.apache.spark.api.java.JavaRDD; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamCarFunction.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamCarFunction.java index 4bb0aea..38af135 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamCarFunction.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamCarFunction.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamIngestionVoidFunction.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamIngestionVoidFunction.java index 45ec447..935b7c2 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamIngestionVoidFunction.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamIngestionVoidFunction.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import java.io.IOException; diff --git a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamVoidFunction.java b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamVoidFunction.java index d3a0f8b..323c3f2 100644 --- a/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamVoidFunction.java +++ b/java-iguazio-api-examples/data-ingestor/src/main/java/com/iguazio/function/StreamVoidFunction.java @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package com.iguazio.function; import java.util.Iterator; diff --git a/netops_demo/Makefile b/netops_demo/Makefile index e2b5e9d..5cdd6b5 100644 --- a/netops_demo/Makefile +++ b/netops_demo/Makefile @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# NETOPS_TAG := $(if $(NETOPS_TAG),$(NETOPS_TAG),latest) NETOPS_REGISTRY_URL := $(if $(NETOPS_REGISTRY_URL),$(NETOPS_REGISTRY_URL),iguaziodocker) diff --git a/netops_demo/golang/src/github.com/v3io/demos/Dockerfile b/netops_demo/golang/src/github.com/v3io/demos/Dockerfile index 0da7224..2b5a3fe 100644 --- a/netops_demo/golang/src/github.com/v3io/demos/Dockerfile +++ b/netops_demo/golang/src/github.com/v3io/demos/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ARG NUCLIO_LABEL=0.5.15 ARG NUCLIO_ARCH=amd64 ARG NUCLIO_BASE_IMAGE=alpine:3.7 diff --git a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/anodot/appender.go b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/anodot/appender.go index 98e669f..6ad5431 100644 --- a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/anodot/appender.go +++ b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/anodot/appender.go @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package anodot import ( diff --git a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest.go b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest.go index 0be906c..bced398 100644 --- a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest.go +++ b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest.go @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package ingest import ( diff --git a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest_test.go b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest_test.go index 7e7b824..c9b0548 100644 --- a/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest_test.go +++ b/netops_demo/golang/src/github.com/v3io/demos/functions/ingest/ingest_test.go @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package ingest import ( diff --git a/netops_demo/golang/src/github.com/v3io/demos/main.go b/netops_demo/golang/src/github.com/v3io/demos/main.go index aa6e4e0..5329c8a 100644 --- a/netops_demo/golang/src/github.com/v3io/demos/main.go +++ b/netops_demo/golang/src/github.com/v3io/demos/main.go @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package main import ( diff --git a/netops_demo/py/Dockerfile b/netops_demo/py/Dockerfile index 69c4845..90ec998 100644 --- a/netops_demo/py/Dockerfile +++ b/netops_demo/py/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ARG NUCLIO_TAG=0.5.15 ARG NUCLIO_ARCH=amd64 ARG NUCLIO_BASE_IMAGE=python:3.7 diff --git a/netops_demo/py/functions/generate/generate.py b/netops_demo/py/functions/generate/generate.py index d29190c..4433909 100644 --- a/netops_demo/py/functions/generate/generate.py +++ b/netops_demo/py/functions/generate/generate.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import os import time import json diff --git a/netops_demo/py/functions/generate/generate_test.py b/netops_demo/py/functions/generate/generate_test.py index df6d20b..cfb6532 100644 --- a/netops_demo/py/functions/generate/generate_test.py +++ b/netops_demo/py/functions/generate/generate_test.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import unittest.mock import os import json diff --git a/netops_demo/py/libs/generator/__init__.py b/netops_demo/py/libs/generator/__init__.py index 366b19d..5b1292e 100644 --- a/netops_demo/py/libs/generator/__init__.py +++ b/netops_demo/py/libs/generator/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from . import metric __all__ = [metric] diff --git a/netops_demo/py/libs/generator/baseline/__init__.py b/netops_demo/py/libs/generator/baseline/__init__.py index 88a5da3..d2514e3 100644 --- a/netops_demo/py/libs/generator/baseline/__init__.py +++ b/netops_demo/py/libs/generator/baseline/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from . import normal __all__ = [normal] diff --git a/netops_demo/py/libs/generator/baseline/company.py b/netops_demo/py/libs/generator/baseline/company.py index 92bb122..69da077 100644 --- a/netops_demo/py/libs/generator/baseline/company.py +++ b/netops_demo/py/libs/generator/baseline/company.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from faker import Faker from libs.generator.baseline.location import LocationProvider from libs.generator.device import Device diff --git a/netops_demo/py/libs/generator/baseline/location.py b/netops_demo/py/libs/generator/baseline/location.py index 38d16cc..cf962cc 100644 --- a/netops_demo/py/libs/generator/baseline/location.py +++ b/netops_demo/py/libs/generator/baseline/location.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from faker.providers import BaseProvider from random import Random from ast import literal_eval as make_tuple diff --git a/netops_demo/py/libs/generator/baseline/normal.py b/netops_demo/py/libs/generator/baseline/normal.py index a88bf74..f58d761 100644 --- a/netops_demo/py/libs/generator/baseline/normal.py +++ b/netops_demo/py/libs/generator/baseline/normal.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import numpy as np def Normal(mu=0, sigma=0.5, noise=0): diff --git a/netops_demo/py/libs/generator/deployment.py b/netops_demo/py/libs/generator/deployment.py index 8db15e2..277b877 100644 --- a/netops_demo/py/libs/generator/deployment.py +++ b/netops_demo/py/libs/generator/deployment.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from libs.generator.baseline.company import Company diff --git a/netops_demo/py/libs/generator/device.py b/netops_demo/py/libs/generator/device.py index 79561e2..190579b 100644 --- a/netops_demo/py/libs/generator/device.py +++ b/netops_demo/py/libs/generator/device.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from libs.generator.metric import Metric from random import Random diff --git a/netops_demo/py/libs/generator/metric.py b/netops_demo/py/libs/generator/metric.py index 0928641..e0735f0 100644 --- a/netops_demo/py/libs/generator/metric.py +++ b/netops_demo/py/libs/generator/metric.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from libs.generator.baseline.normal import Normal from random import Random diff --git a/netops_demo/py/libs/nuclio_sdk/exceptions.py b/netops_demo/py/libs/nuclio_sdk/exceptions.py index 0b5d7d8..f17ada5 100644 --- a/netops_demo/py/libs/nuclio_sdk/exceptions.py +++ b/netops_demo/py/libs/nuclio_sdk/exceptions.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# class ExceptionWithResponse(IOError): def __init__(self, status_code, body=None, content_type=None): diff --git a/netops_demo/py/run_generator.py b/netops_demo/py/run_generator.py index 714ce29..e9ec970 100644 --- a/netops_demo/py/run_generator.py +++ b/netops_demo/py/run_generator.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #from generator.manager import Manager import libs.generator.deployment import json diff --git a/taxi_nosql/Makefile b/taxi_nosql/Makefile index 9da2666..c5c411b 100644 --- a/taxi_nosql/Makefile +++ b/taxi_nosql/Makefile @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# .PHONY: python-demo python-demo: docker build -f nuclio_python/Dockerfile -t cloud_demo_functions:latest . diff --git a/taxi_nosql/create_random_driver_data.py b/taxi_nosql/create_random_driver_data.py index f69c79f..44c86b9 100644 --- a/taxi_nosql/create_random_driver_data.py +++ b/taxi_nosql/create_random_driver_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from decimal import * from random import * diff --git a/taxi_nosql/create_random_driver_data.sh b/taxi_nosql/create_random_driver_data.sh index 5e8e3d0..573d6c2 100755 --- a/taxi_nosql/create_random_driver_data.sh +++ b/taxi_nosql/create_random_driver_data.sh @@ -1 +1,15 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# python create_random_driver_data.py > drivers_data.csv diff --git a/taxi_nosql/create_random_passenger_data.py b/taxi_nosql/create_random_passenger_data.py index faeb869..bf9f54b 100644 --- a/taxi_nosql/create_random_passenger_data.py +++ b/taxi_nosql/create_random_passenger_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from decimal import * from random import * diff --git a/taxi_nosql/create_random_passenger_data.sh b/taxi_nosql/create_random_passenger_data.sh index 387f62c..f4dc4d9 100644 --- a/taxi_nosql/create_random_passenger_data.sh +++ b/taxi_nosql/create_random_passenger_data.sh @@ -1 +1,15 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# python create_random_passenger_data.py > passengers_data.csv diff --git a/taxi_nosql/igz_nosql_web.py b/taxi_nosql/igz_nosql_web.py index 7a8bf8d..c9c7b2d 100644 --- a/taxi_nosql/igz_nosql_web.py +++ b/taxi_nosql/igz_nosql_web.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import requests import json diff --git a/taxi_nosql/load_driver_data.py b/taxi_nosql/load_driver_data.py index 43806fa..6f6396b 100755 --- a/taxi_nosql/load_driver_data.py +++ b/taxi_nosql/load_driver_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import sys import requests import csv diff --git a/taxi_nosql/load_driver_data.sh b/taxi_nosql/load_driver_data.sh index 62a164a..17378d0 100755 --- a/taxi_nosql/load_driver_data.sh +++ b/taxi_nosql/load_driver_data.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/bin/bash source set_env.sh python load_driver_data.py drivers_data.csv diff --git a/taxi_nosql/load_passenger_data.py b/taxi_nosql/load_passenger_data.py index 7304331..6ee14fd 100755 --- a/taxi_nosql/load_passenger_data.py +++ b/taxi_nosql/load_passenger_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import sys import requests import csv diff --git a/taxi_nosql/load_passenger_data.sh b/taxi_nosql/load_passenger_data.sh index faf2a2e..11d7018 100755 --- a/taxi_nosql/load_passenger_data.sh +++ b/taxi_nosql/load_passenger_data.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/bin/bash source set_env.sh python load_passenger_data.py passengers_data.csv diff --git a/taxi_nosql/nuclio_golang/DriverKV.go b/taxi_nosql/nuclio_golang/DriverKV.go index 659f623..df6504c 100644 --- a/taxi_nosql/nuclio_golang/DriverKV.go +++ b/taxi_nosql/nuclio_golang/DriverKV.go @@ -1,3 +1,22 @@ +/* +Copyright 2017 Iguazio Systems Ltd. + +Licensed under the Apache License, Version 2.0 (the "License") with +an addition restriction as set forth herein. You may not use this +file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. + +In addition, you may not use the software for any purposes that are +illegal under applicable law, and the grant of the foregoing license +under the Apache 2.0 license is conditioned upon your compliance with +such restriction. +*/ package main import ( diff --git a/taxi_nosql/nuclio_golang/function.yaml b/taxi_nosql/nuclio_golang/function.yaml index 37979fa..c7da588 100644 --- a/taxi_nosql/nuclio_golang/function.yaml +++ b/taxi_nosql/nuclio_golang/function.yaml @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# metadata: name: driverkv labels: {} diff --git a/taxi_nosql/nuclio_golang/insert_driver_nuclio.py b/taxi_nosql/nuclio_golang/insert_driver_nuclio.py index 1eae8d1..f9468ac 100644 --- a/taxi_nosql/nuclio_golang/insert_driver_nuclio.py +++ b/taxi_nosql/nuclio_golang/insert_driver_nuclio.py @@ -1,41 +1,55 @@ -import sys -import requests -import json -import csv -import time - -#------------ -BASE_URL = 'http://127.0.0.1:31223' - -# read CSV -INPUT_FILE = str(sys.argv[1]) - -start = time.time() -counter = 0 -s = requests.Session() -with open(INPUT_FILE) as csvfile: - readCSV = csv.reader(csvfile, delimiter=',') - # Skip the header - next(readCSV, None) - # Go over the rows and get the driver id and cell id - for row in readCSV: - driver_id = row[0] - time_stamp = row[1] - lat = row[2] - long = row[3] - status = row[4] - body = driver_id + ',' + time_stamp + ',' + lat + ',' + long + ',' + status - #print(body) - - # call the update request - res = s.put(BASE_URL, data=body, headers=None) - if res.status_code == requests.codes.bad_request: - print(res.content) - print(res.status_code) - - counter = counter + 1 - if counter % 1000 == 0: - end = time.time() - print("File: {}, timing: {}, Counter: {}".format(INPUT_FILE, end - start, counter)) -end = time.time() -print("Total File: {}, timing: {}, Counter: {}".format(INPUT_FILE, end - start, counter)) +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys +import requests +import json +import csv +import time + +#------------ +BASE_URL = 'http://127.0.0.1:31223' + +# read CSV +INPUT_FILE = str(sys.argv[1]) + +start = time.time() +counter = 0 +s = requests.Session() +with open(INPUT_FILE) as csvfile: + readCSV = csv.reader(csvfile, delimiter=',') + # Skip the header + next(readCSV, None) + # Go over the rows and get the driver id and cell id + for row in readCSV: + driver_id = row[0] + time_stamp = row[1] + lat = row[2] + long = row[3] + status = row[4] + body = driver_id + ',' + time_stamp + ',' + lat + ',' + long + ',' + status + #print(body) + + # call the update request + res = s.put(BASE_URL, data=body, headers=None) + if res.status_code == requests.codes.bad_request: + print(res.content) + print(res.status_code) + + counter = counter + 1 + if counter % 1000 == 0: + end = time.time() + print("File: {}, timing: {}, Counter: {}".format(INPUT_FILE, end - start, counter)) +end = time.time() +print("Total File: {}, timing: {}, Counter: {}".format(INPUT_FILE, end - start, counter)) diff --git a/taxi_nosql/nuclio_python/Dockerfile b/taxi_nosql/nuclio_python/Dockerfile index c4b7f99..e9a57f5 100644 --- a/taxi_nosql/nuclio_python/Dockerfile +++ b/taxi_nosql/nuclio_python/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ARG NUCLIO_TAG=0.5.3 ARG NUCLIO_ARCH=amd64 ARG NUCLIO_BASE_IMAGE=python:3.6-alpine diff --git a/taxi_nosql/nuclio_python/functions/generate_data/generate_data.py b/taxi_nosql/nuclio_python/functions/generate_data/generate_data.py index 74055e8..d2783b9 100644 --- a/taxi_nosql/nuclio_python/functions/generate_data/generate_data.py +++ b/taxi_nosql/nuclio_python/functions/generate_data/generate_data.py @@ -1,118 +1,132 @@ -import requests -import os -import json -import random -from decimal import * - -# Get the ingestion-function URL from the Nuclio function environment variables -INGEST_URL = os.getenv("INGEST_URL") - -# List of GPS coordinates for randomly selected locations -locations = { - 'downtown_london': {'long': -0.1195, 'lat': 51.5033}, - 'westminster': {'long': -0.1357, 'lat': 51.4975}, - 'oxford_st': {'long': -0.1410, 'lat': 51.5154}, - 'heathrow': {'long': -0.4543, 'lat': 51.4700}, - 'heathrow_parking': {'long': -0.4473, 'lat': 51.4599}, - 'gatwick': {'long': 0.1821, 'lat': 51.1537}, - 'canary_wharf': {'long': -0.0235, 'lat': 51.5054} -} - -# Most drivers are in downtown - provide more weight to downtown. -# Note that this is just an example. -drivers_weighted_locations = {'downtown_london': 4, 'westminster': 3} - -# Most passengers are in airports - provide more weight to airports. -# Note that this is just an example. -passengers_weighted_locations = \ - {'heathrow': 4, 'heathrow_parking': 3, 'gatwick': 4} - -# Set the number of drivers and passengers to ingest -num_drivers_to_ingest = 1000 -num_passengers_to_ingest = 500 - -# Set the maximum valid driver and passenger ID values -max_driver_id = 5000 -max_passenger_id = 5000 - - -# Function handler - generate random drivers, passengers, and locations data -# and send it to the configured ingestion URL (INGEST_URL) -def handler(context, event): - - # Ingest current driver locations - _ingest_locations(context, num_drivers_to_ingest, max_driver_id, 'driver', - drivers_weighted_locations) - - # Ingest current passenger locations - _ingest_locations(context, num_passengers_to_ingest, max_passenger_id, - 'passenger', passengers_weighted_locations) - - return context.Response(status_code=204) - - -# Ingest driver and passenger locations information -def _ingest_locations(context, num_records, max_record_id, record_type, - weighted_locations): - - # Get random driver/passenger locations and send the data for ingestion - for x in range(1, num_records): - - # Get a weighted random location - random_location = _get_random_location(_weighted_keys(locations, - weighted_locations)) - - # Construct the request's JSON body. The body includes the record type - # (driver/passenger), the respective ID, and the GPS coordinates of the - # current driver or passenger location. - # For demo purposes, the location coordinates are generated as a random - # offset of the current random weighted location. - request_json = { - 'RecordType': record_type, - 'ID': random.randint(1, max_record_id), - 'Longitude': - _get_random_offset(locations[random_location]['long']), - 'Latitude': _get_random_offset(locations[random_location]['lat']) - } - - # Ingest the location data by sending an HTTP request to the configured - # ingestion URL - response = requests.put(INGEST_URL, data=json.dumps(request_json)) - - if response.status_code != requests.codes.ok: - message = f'''Ingestion of {record_type}s failed with error code - {response.status_code}''' - context.logger.error(message) - return context.Response(body={'error': message}, status_code=500) - - -# Get a random location from the available locations -def _get_random_location(locations): - - random_location_name = random.choice(locations) - - return random_location_name - - -# Generate a close random offset of the given location - for demo purposes only -def _get_random_offset(location): - - rnd_radius = random.randint(0, 8) - rnd_offset = random.randint(-rnd_radius, rnd_radius) - new_location = Decimal(location) + (Decimal(rnd_offset) / Decimal(300)) - - return str(new_location) - - -# Get weighted keys: given dictionary "d", return the dictionary keys weighted -# by weight "weights". For example: -# input: d = {'a': 1, 'b': 2, 'c': 3}, weights = {'b': 3, 'c': 2} -# output: ['a', 'b', 'b', 'b', 'c', 'c'] -def _weighted_keys(d, weights): - result = [] - - for key in d.keys(): - result.extend([key] * weights.get(key, 1)) - - return result - +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import requests +import os +import json +import random +from decimal import * + +# Get the ingestion-function URL from the Nuclio function environment variables +INGEST_URL = os.getenv("INGEST_URL") + +# List of GPS coordinates for randomly selected locations +locations = { + 'downtown_london': {'long': -0.1195, 'lat': 51.5033}, + 'westminster': {'long': -0.1357, 'lat': 51.4975}, + 'oxford_st': {'long': -0.1410, 'lat': 51.5154}, + 'heathrow': {'long': -0.4543, 'lat': 51.4700}, + 'heathrow_parking': {'long': -0.4473, 'lat': 51.4599}, + 'gatwick': {'long': 0.1821, 'lat': 51.1537}, + 'canary_wharf': {'long': -0.0235, 'lat': 51.5054} +} + +# Most drivers are in downtown - provide more weight to downtown. +# Note that this is just an example. +drivers_weighted_locations = {'downtown_london': 4, 'westminster': 3} + +# Most passengers are in airports - provide more weight to airports. +# Note that this is just an example. +passengers_weighted_locations = \ + {'heathrow': 4, 'heathrow_parking': 3, 'gatwick': 4} + +# Set the number of drivers and passengers to ingest +num_drivers_to_ingest = 1000 +num_passengers_to_ingest = 500 + +# Set the maximum valid driver and passenger ID values +max_driver_id = 5000 +max_passenger_id = 5000 + + +# Function handler - generate random drivers, passengers, and locations data +# and send it to the configured ingestion URL (INGEST_URL) +def handler(context, event): + + # Ingest current driver locations + _ingest_locations(context, num_drivers_to_ingest, max_driver_id, 'driver', + drivers_weighted_locations) + + # Ingest current passenger locations + _ingest_locations(context, num_passengers_to_ingest, max_passenger_id, + 'passenger', passengers_weighted_locations) + + return context.Response(status_code=204) + + +# Ingest driver and passenger locations information +def _ingest_locations(context, num_records, max_record_id, record_type, + weighted_locations): + + # Get random driver/passenger locations and send the data for ingestion + for x in range(1, num_records): + + # Get a weighted random location + random_location = _get_random_location(_weighted_keys(locations, + weighted_locations)) + + # Construct the request's JSON body. The body includes the record type + # (driver/passenger), the respective ID, and the GPS coordinates of the + # current driver or passenger location. + # For demo purposes, the location coordinates are generated as a random + # offset of the current random weighted location. + request_json = { + 'RecordType': record_type, + 'ID': random.randint(1, max_record_id), + 'Longitude': + _get_random_offset(locations[random_location]['long']), + 'Latitude': _get_random_offset(locations[random_location]['lat']) + } + + # Ingest the location data by sending an HTTP request to the configured + # ingestion URL + response = requests.put(INGEST_URL, data=json.dumps(request_json)) + + if response.status_code != requests.codes.ok: + message = f'''Ingestion of {record_type}s failed with error code + {response.status_code}''' + context.logger.error(message) + return context.Response(body={'error': message}, status_code=500) + + +# Get a random location from the available locations +def _get_random_location(locations): + + random_location_name = random.choice(locations) + + return random_location_name + + +# Generate a close random offset of the given location - for demo purposes only +def _get_random_offset(location): + + rnd_radius = random.randint(0, 8) + rnd_offset = random.randint(-rnd_radius, rnd_radius) + new_location = Decimal(location) + (Decimal(rnd_offset) / Decimal(300)) + + return str(new_location) + + +# Get weighted keys: given dictionary "d", return the dictionary keys weighted +# by weight "weights". For example: +# input: d = {'a': 1, 'b': 2, 'c': 3}, weights = {'b': 3, 'c': 2} +# output: ['a', 'b', 'b', 'b', 'c', 'c'] +def _weighted_keys(d, weights): + result = [] + + for key in d.keys(): + result.extend([key] * weights.get(key, 1)) + + return result + diff --git a/taxi_nosql/nuclio_python/functions/ingest/ingest.py b/taxi_nosql/nuclio_python/functions/ingest/ingest.py index 7ec45a7..4e68ed1 100644 --- a/taxi_nosql/nuclio_python/functions/ingest/ingest.py +++ b/taxi_nosql/nuclio_python/functions/ingest/ingest.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import requests import json import os diff --git a/taxi_nosql/nuclio_python/generate_data.yaml b/taxi_nosql/nuclio_python/generate_data.yaml index 202fbd6..c1f2be3 100644 --- a/taxi_nosql/nuclio_python/generate_data.yaml +++ b/taxi_nosql/nuclio_python/generate_data.yaml @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# metadata: name: generate_data labels: diff --git a/taxi_nosql/nuclio_python/ingest.yaml b/taxi_nosql/nuclio_python/ingest.yaml index 9e46d76..0c8235d 100644 --- a/taxi_nosql/nuclio_python/ingest.yaml +++ b/taxi_nosql/nuclio_python/ingest.yaml @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# kind: Function metadata: name: ingest diff --git a/taxi_nosql/remove_all_data.sh b/taxi_nosql/remove_all_data.sh index 1418fb3..0180d31 100644 --- a/taxi_nosql/remove_all_data.sh +++ b/taxi_nosql/remove_all_data.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/bin/bash source set_env.sh hdfs dfs -rm -r $DRIVERS_TABLE diff --git a/taxi_nosql/set_env.sh b/taxi_nosql/set_env.sh index 89c825e..6da0e9c 100755 --- a/taxi_nosql/set_env.sh +++ b/taxi_nosql/set_env.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# export WEBAPI_URL="http://127.0.0.1:8081" export CONTAINER_NAME="/bigdata" export DRIVERS_TABLE="/taxi_example/drivers/" diff --git a/taxi_nosql/try.py b/taxi_nosql/try.py index dc52c83..bd0f637 100644 --- a/taxi_nosql/try.py +++ b/taxi_nosql/try.py @@ -1 +1,15 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# uuuu##test1 diff --git a/taxi_streaming/consume_drivers_stream_data.py b/taxi_streaming/consume_drivers_stream_data.py index b33c346..120d8b3 100755 --- a/taxi_streaming/consume_drivers_stream_data.py +++ b/taxi_streaming/consume_drivers_stream_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from pyspark.sql import SparkSession from pyspark.streaming import StreamingContext from v3io.spark.streaming import * diff --git a/taxi_streaming/create_drivers_stream.sh b/taxi_streaming/create_drivers_stream.sh index 26c7971..1d615b0 100644 --- a/taxi_streaming/create_drivers_stream.sh +++ b/taxi_streaming/create_drivers_stream.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash # IP address of the platform's web-gateway service diff --git a/taxi_streaming/create_random_drivers_data.py b/taxi_streaming/create_random_drivers_data.py index ba52514..23ec8fe 100644 --- a/taxi_streaming/create_random_drivers_data.py +++ b/taxi_streaming/create_random_drivers_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # Generate random drivers stream data from decimal import * diff --git a/taxi_streaming/create_random_drivers_data.sh b/taxi_streaming/create_random_drivers_data.sh index 02bce35..b52d7b3 100755 --- a/taxi_streaming/create_random_drivers_data.sh +++ b/taxi_streaming/create_random_drivers_data.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash # Stream-data input file diff --git a/taxi_streaming/delete_stream.sh b/taxi_streaming/delete_stream.sh index 3ba0856..300aec2 100644 --- a/taxi_streaming/delete_stream.sh +++ b/taxi_streaming/delete_stream.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash # Data-container name diff --git a/taxi_streaming/spark-submit_cmd.sh b/taxi_streaming/spark-submit_cmd.sh index 450f965..6bceb3a 100755 --- a/taxi_streaming/spark-submit_cmd.sh +++ b/taxi_streaming/spark-submit_cmd.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash # Submit a spark job to consume the stream data diff --git a/taxi_streaming/stream_drivers_data.py b/taxi_streaming/stream_drivers_data.py index 1354861..76a9297 100644 --- a/taxi_streaming/stream_drivers_data.py +++ b/taxi_streaming/stream_drivers_data.py @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import base64 import collections import json diff --git a/taxi_streaming/stream_drivers_data.sh b/taxi_streaming/stream_drivers_data.sh index 1f84934..805a86b 100755 --- a/taxi_streaming/stream_drivers_data.sh +++ b/taxi_streaming/stream_drivers_data.sh @@ -1,3 +1,17 @@ +# Copyright 2017 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #!/usr/bin/env bash # Stream-data input file