From ee6b5a8f7edcc179d78a69d4bbfc70f714754a3f Mon Sep 17 00:00:00 2001 From: telliere Date: Thu, 21 Mar 2024 11:47:01 +0200 Subject: [PATCH] introducing configuration file for ship_a_key.py --- client/container_preparation/entrypoint.sh | 9 ++-- client/data_preparation/entrypoint.sh | 9 ++-- utils/ship_a_key.py | 56 ++++++++++++++-------- utils/ssh_utils.py | 6 ++- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh index 56d4c74..099532f 100755 --- a/client/container_preparation/entrypoint.sh +++ b/client/container_preparation/entrypoint.sh @@ -10,6 +10,7 @@ docker_path="/var/run/docker.sock" parse_args() { while [[ "$#" -gt 0 ]]; do case "$1" in + --config) config="$2"; shift 2 ;; -b|--base-oci-image) base_oci_image="$2"; shift 2 ;; -s|--sif-path) sif_path="$2"; shift 2 ;; -e|--encrypted) encrypted=true; shift ;; @@ -26,7 +27,7 @@ parse_args() { done # Check for required arguments - if [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then + if [ -z "$config" ] || [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then echo echo "Please provides options for both of these programs : " python3 ./prepare_container.py --help python3 ./utils/ship_a_key.py --help @@ -110,13 +111,13 @@ else if [ -z "$users" ]; then # If the user provided only groups - python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 elif [ -z "$groups" ] ; then # If the user provided only users - python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 else # If the user provided both - python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 fi echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Key written to the vault" diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh index 1627cee..ffbb7f8 100755 --- a/client/data_preparation/entrypoint.sh +++ b/client/data_preparation/entrypoint.sh @@ -7,6 +7,7 @@ parse_args() { while [[ "$#" -gt 0 ]]; do case "$1" in + --config) config="$2"; shift 2 ;; -i|--input-data) input_data="$2"; shift 2 ;; -o|--output-data) output_data="$2"; shift 2 ;; --data-path) data_path="$2"; shift 2 ;; @@ -21,7 +22,7 @@ parse_args() { done # Check for required arguments - if [ -z "$input_data" ] || [ -z "$output_data" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || [ -z "$username" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then + if [ -z "$config" ] || [ -z "$input_data" ] || [ -z "$output_data" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || [ -z "$username" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then echo echo "Please provides options for both of these programs : " python3 ./prepare_data.py --help python3 ./utils/ship_a_key.py --help @@ -96,13 +97,13 @@ echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Writing key to th # Handle different cases of user provided compute nodes / user / groups if [ -z "$users" ]; then # If the user provided only groups - python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 elif [ -z "$groups" ] ; then # If the user provided only users - python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 else # If the user provided both - python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 + python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1 fi echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Key written to the vault" diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py index affce34..d56e5da 100644 --- a/utils/ship_a_key.py +++ b/utils/ship_a_key.py @@ -11,6 +11,8 @@ from hashlib import sha512 from ssh_utils import ssh_connect, ssh_copy_file +from configparser import ConfigParser, NoSectionError, NoOptionError + # Provide client_id from cli$ # Same for trust domain # Get image id and transform as for server @@ -26,6 +28,11 @@ def parse_arguments() -> argparse.ArgumentParser: """ parser = argparse.ArgumentParser(description="CLI Options") + parser.add_argument( + "--config", + required=True, + help="Path to the client configuration file", + ) parser.add_argument( "--users", "-u", @@ -82,19 +89,6 @@ def parse_arguments() -> argparse.ArgumentParser: type=str, help="Path to write the dataset on the supercomputer storage default :", ) - parser.add_argument( - "--sd-server-address", - "-a", - type=str, - help="Server address", - ) - parser.add_argument( - "--sd-server-port", - "-ap", - type=int, - default=10080, - help="SD API server port (default: 10080)", - ) parser.add_argument( "--username", required=True, @@ -103,6 +97,25 @@ def parse_arguments() -> argparse.ArgumentParser: return parser.parse_args() +# Parse configuration file +def parse_configuration(path : str): + config = ConfigParser() + config.read(path) + + if not 'hpcs-server' in config: + raise NoSectionError("hpcs-server section missing in configuration file, aborting") + + if not 'vault' in config: + raise NoSectionError("vault section missing in configuration file, aborting") + + if not 'url' in config['hpcs-server']: + raise NoOptionError("'hpcs-server' section is incomplete in configuration file, aborting") + + if not 'url' in config['vault']: + raise NoOptionError("'vault' section is incomplete in configuration file, aborting") + + return config + def validate_options(options: argparse.ArgumentParser): """Check for the cli-provided options @@ -194,7 +207,7 @@ def validate_options(options: argparse.ArgumentParser): def create_authorized_workloads( - SVID: JwtSvid, secret, server, port, users, groups, compute_nodes + SVID: JwtSvid, secret, url, users, groups, compute_nodes ): """Create workloads that are authorized to access to a secret @@ -212,7 +225,7 @@ def create_authorized_workloads( """ # Prepare request - url = f"http://{server}:{port}/api/client/create-workloads" + url = f"{url}/api/client/create-workloads" payload = { "jwt": SVID.token, "secret": secret, @@ -248,7 +261,9 @@ def create_authorized_workloads( if __name__ == "__main__": # Parse arguments from CLI - options = parse_arguments() + options = parse_arguments() + # Parse configuration file + configuration = parse_configuration(options.config) # Validate / Parse them ( @@ -277,15 +292,14 @@ def create_authorized_workloads( users_spiffeID, client_id, secrets_path, user_role = create_authorized_workloads( SVID, secret_name, - options.sd_server_address, - options.sd_server_port, + configuration["hpcs-server"]["url"], users, groups, compute_nodes, ) # Login to the vault using client's certificate - hvac_client = vault_login(SVID, f"client_{client_id}") + hvac_client = vault_login(configuration["vault"]["url"], SVID, f"client_{client_id}") # Prepare secret secret = {} @@ -329,11 +343,11 @@ def create_authorized_workloads( ssh_copy_file( ssh_client, "/tmp/dataset_info.yaml", - f"{options.data_path_at_rest}{secret_name}.info.yaml", + f"{options.data_path_at_rest}/{secret_name}.info.yaml", ) print( - f"Data and info file were shipped to te supercomputer. Infos about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml" + f"Data and info file were shipped to te supercomputer. Info about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml" ) ssh_client.close() diff --git a/utils/ssh_utils.py b/utils/ssh_utils.py index ade6b5f..90eb3d6 100644 --- a/utils/ssh_utils.py +++ b/utils/ssh_utils.py @@ -1,5 +1,5 @@ from paramiko.client import SSHClient -from paramiko import SSHException, AutoAddPolicy +from paramiko import SSHException, AutoAddPolicy, RSAKey from scp import SCPClient # Hostname and port configuration @@ -30,11 +30,13 @@ def ssh_connect(username: str) -> SSHClient: # Probably running in a container except SSHException: + pkey=RSAKey.from_private_key_file("/tmp/.ssh/id_rsa") client.connect( host, port, username=username, - key_filename="/tmp/.ssh/id_rsa", + pkey=pkey, + look_for_keys=False, auth_timeout=30, timeout=30, )