From d09efae454f6bf4bac185489b1d18b2cc5042cf9 Mon Sep 17 00:00:00 2001 From: Lauren Hughes Date: Thu, 3 Jan 2019 14:48:14 -0800 Subject: [PATCH 1/2] Update requirements.txt Update to latest version --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index aa15d6a..c195619 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,2 +1,2 @@ -azure-batch==5.1.1 +azure-batch==6.0.0 azure-storage-blob==1.4.0 From decf323c0af6cf8eb07167d9d01a255f2b57f463 Mon Sep 17 00:00:00 2001 From: Lauren Hughes Date: Mon, 14 Jan 2019 11:37:01 -0800 Subject: [PATCH 2/2] Update SDK to latest, add config file --- src/batch_python_tutorial_ffmpeg.py | 51 +++++++++++------------------ src/config.py | 31 ++++++++++++++++++ src/requirements.txt | 4 +-- 3 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 src/config.py diff --git a/src/batch_python_tutorial_ffmpeg.py b/src/batch_python_tutorial_ffmpeg.py index ade4219..1a7095d 100644 --- a/src/batch_python_tutorial_ffmpeg.py +++ b/src/batch_python_tutorial_ffmpeg.py @@ -4,6 +4,7 @@ import os import sys import time +import config try: input = raw_input @@ -18,24 +19,10 @@ sys.path.append('.') sys.path.append('..') - -# Update the Batch and Storage account credential strings below with the values +# Update the Batch and Storage account credential strings in config.py with values # unique to your accounts. These are used when constructing connection strings # for the Batch and Storage client objects. -# global -_BATCH_ACCOUNT_NAME ='' -_BATCH_ACCOUNT_KEY = '' -_BATCH_ACCOUNT_URL = '' -_STORAGE_ACCOUNT_NAME = '' -_STORAGE_ACCOUNT_KEY = '' -_POOL_ID = 'LinuxFfmpegPool' -_DEDICATED_POOL_NODE_COUNT = 0 -_LOW_PRIORITY_POOL_NODE_COUNT = 5 -_POOL_VM_SIZE = 'STANDARD_A1_v2' -_JOB_ID = 'LinuxFfmpegJob' - - def query_yes_no(question, default="yes"): """ Prompts the user for yes/no input, displaying the specified question text. @@ -116,7 +103,7 @@ def upload_file_to_container(block_blob_client, container_name, file_path): sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, - blob_source=sas_url) + http_url=sas_url) def get_container_sas_token(block_blob_client, container_name, blob_permissions): @@ -162,7 +149,7 @@ def get_container_sas_url(block_blob_client, container_name, azureblob.BlobPermissions.WRITE) # Construct SAS URL for the container - container_sas_url = "https://{}.blob.core.windows.net/{}?{}".format(_STORAGE_ACCOUNT_NAME, container_name, sas_token) + container_sas_url = "https://{}.blob.core.windows.net/{}?{}".format(config._STORAGE_ACCOUNT_NAME, container_name, sas_token) return container_sas_url @@ -198,9 +185,9 @@ def create_pool(batch_service_client, pool_id): version="latest" ), node_agent_sku_id="batch.node.ubuntu 18.04"), - vm_size=_POOL_VM_SIZE, - target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT, - target_low_priority_nodes=_LOW_PRIORITY_POOL_NODE_COUNT, + vm_size=config._POOL_VM_SIZE, + target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT, + target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT, start_task=batchmodels.StartTask( command_line="/bin/bash -c \"apt-get update && apt-get install -y ffmpeg\"", wait_for_success=True, @@ -314,8 +301,8 @@ def wait_for_tasks_to_complete(batch_service_client, job_id, timeout): blob_client = azureblob.BlockBlobService( - account_name=_STORAGE_ACCOUNT_NAME, - account_key=_STORAGE_ACCOUNT_KEY) + account_name=config._STORAGE_ACCOUNT_NAME, + account_key=config._STORAGE_ACCOUNT_KEY) # Use the blob client to create the containers in Azure Storage if they # don't yet exist. @@ -350,29 +337,29 @@ def wait_for_tasks_to_complete(batch_service_client, job_id, timeout): # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage - credentials = batchauth.SharedKeyCredentials(_BATCH_ACCOUNT_NAME, - _BATCH_ACCOUNT_KEY) + credentials = batchauth.SharedKeyCredentials(config._BATCH_ACCOUNT_NAME, + config._BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient( credentials, - base_url=_BATCH_ACCOUNT_URL) + batch_url=config._BATCH_ACCOUNT_URL) try: # Create the pool that will contain the compute nodes that will execute the # tasks. - create_pool(batch_client, _POOL_ID) + create_pool(batch_client, config._POOL_ID) # Create the job that will run the tasks. - create_job(batch_client, _JOB_ID, _POOL_ID) + create_job(batch_client, config._JOB_ID, config._POOL_ID) # Add the tasks to the job. Pass the input files and a SAS URL # to the storage container for output files. - add_tasks(batch_client, _JOB_ID, input_files, output_container_sas_url) + add_tasks(batch_client, config._JOB_ID, input_files, output_container_sas_url) # Pause execution until tasks reach Completed state. wait_for_tasks_to_complete(batch_client, - _JOB_ID, - datetime.timedelta(minutes=30)) + config._JOB_ID, + datetime.timedelta(minutes=30)) print(" Success! All tasks reached the 'Completed' state within the " "specified timeout period.") @@ -395,10 +382,10 @@ def wait_for_tasks_to_complete(batch_service_client, job_id, timeout): # Clean up Batch resources (if the user so chooses). if query_yes_no('Delete job?') == 'yes': - batch_client.job.delete(_JOB_ID) + batch_client.job.delete(config._JOB_ID) if query_yes_no('Delete pool?') == 'yes': - batch_client.pool.delete(_POOL_ID) + batch_client.pool.delete(config._POOL_ID) print() input('Press ENTER to exit...') diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..b90bdec --- /dev/null +++ b/src/config.py @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------- +# +# THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. +#---------------------------------------------------------------------------------- +# The example companies, organizations, products, domain names, +# e-mail addresses, logos, people, places, and events depicted +# herein are fictitious. No association with any real company, +# organization, product, domain name, email address, logo, person, +# places, or events is intended or should be inferred. +#-------------------------------------------------------------------------- + +# Global constant variables (Azure Storage account/Batch details) + +# import "config.py" in "batch_python_tutorial_ffmpeg.py" + +# Update the Batch and Storage account credential strings below with the values +# unique to your accounts. These are used when constructing connection strings +# for the Batch and Storage client objects. + +_BATCH_ACCOUNT_NAME ='' +_BATCH_ACCOUNT_KEY = '' +_BATCH_ACCOUNT_URL = '' +_STORAGE_ACCOUNT_NAME = '' +_STORAGE_ACCOUNT_KEY = '' +_POOL_ID = 'LinuxFfmpegPool' +_DEDICATED_POOL_NODE_COUNT = 0 +_LOW_PRIORITY_POOL_NODE_COUNT = 5 +_POOL_VM_SIZE = 'STANDARD_A1_v2' +_JOB_ID = 'LinuxFfmpegJob' \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 004b96c..f1e798d 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,2 +1,2 @@ -azure-batch==4.0.0 -azure-storage-blob==1.3.1 \ No newline at end of file +azure-batch==6.0.0 +azure-storage-blob==1.4.0 \ No newline at end of file