introducing configuration file for ship_a_key.py

client/container_preparation/entrypoint.sh

@@ -10,6 +10,7 @@ docker_path="/var/run/docker.sock"
 parse_args() {
  while [[ "$#" -gt 0 ]]; do
  case "$1" in
+ --config) config="$2"; shift 2 ;;
  -b|--base-oci-image) base_oci_image="$2"; shift 2 ;;
  -s|--sif-path) sif_path="$2"; shift 2 ;;
  -e|--encrypted) encrypted=true; shift ;;
@@ -26,7 +27,7 @@ parse_args() {
  done
 
  # Check for required arguments
- if [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
+ if [ -z "$config" ] || [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
  echo echo "Please provides options for both of these programs : "
  python3 ./prepare_container.py --help
  python3 ./utils/ship_a_key.py --help
@@ -110,13 +111,13 @@ else
 
  if [ -z "$users" ]; then
  # If the user provided only groups
- python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
  elif [ -z "$groups" ] ; then
  # If the user provided only users
- python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
  else
  # If the user provided both
- python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
  fi
 
  echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Key written to the vault"

client/data_preparation/entrypoint.sh

@@ -7,6 +7,7 @@
 parse_args() {
  while [[ "$#" -gt 0 ]]; do
  case "$1" in
+ --config) config="$2"; shift 2 ;;
  -i|--input-data) input_data="$2"; shift 2 ;;
  -o|--output-data) output_data="$2"; shift 2 ;;
  --data-path) data_path="$2"; shift 2 ;;
@@ -21,7 +22,7 @@ parse_args() {
  done
 
  # Check for required arguments
- if [ -z "$input_data" ] || [ -z "$output_data" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || [ -z "$username" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
+ if [ -z "$config" ] || [ -z "$input_data" ] || [ -z "$output_data" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || [ -z "$username" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
  echo echo "Please provides options for both of these programs : "
  python3 ./prepare_data.py --help
  python3 ./utils/ship_a_key.py --help
@@ -96,13 +97,13 @@ echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Writing key to th
 # Handle different cases of user provided compute nodes / user / groups
 if [ -z "$users" ]; then
  # If the user provided only groups
- python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
 elif [ -z "$groups" ] ; then
  # If the user provided only users
- python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
 else
  # If the user provided both
- python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+ python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
 fi
 
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Key written to the vault"

utils/ship_a_key.py

@@ -11,6 +11,8 @@
 from hashlib import sha512
 from ssh_utils import ssh_connect, ssh_copy_file
 
+from configparser import ConfigParser, NoSectionError, NoOptionError
+
 # Provide client_id from cli$
 # Same for trust domain
 # Get image id and transform as for server
@@ -26,6 +28,11 @@ def parse_arguments() -> argparse.ArgumentParser:
  """
  parser = argparse.ArgumentParser(description="CLI Options")
 
+ parser.add_argument(
+ "--config",
+ required=True,
+ help="Path to the client configuration file",
+ )
  parser.add_argument(
  "--users",
  "-u",
@@ -82,19 +89,6 @@ def parse_arguments() -> argparse.ArgumentParser:
  type=str,
  help="Path to write the dataset on the supercomputer storage default :",
  )
- parser.add_argument(
- "--sd-server-address",
- "-a",
- type=str,
- help="Server address",
- )
- parser.add_argument(
- "--sd-server-port",
- "-ap",
- type=int,
- default=10080,
- help="SD API server port (default: 10080)",
- )
  parser.add_argument(
  "--username",
  required=True,
@@ -103,6 +97,25 @@ def parse_arguments() -> argparse.ArgumentParser:
 
  return parser.parse_args()
 
+# Parse configuration file
+def parse_configuration(path : str):
+ config = ConfigParser()
+ config.read(path)
+
+ if not 'hpcs-server' in config:
+ raise NoSectionError("hpcs-server section missing in configuration file, aborting")
+
+ if not 'vault' in config:
+ raise NoSectionError("vault section missing in configuration file, aborting")
+
+ if not 'url' in config['hpcs-server']:
+ raise NoOptionError("'hpcs-server' section is incomplete in configuration file, aborting")
+
+ if not 'url' in config['vault']:
+ raise NoOptionError("'vault' section is incomplete in configuration file, aborting")
+
+ return config
+
 
 def validate_options(options: argparse.ArgumentParser):
  """Check for the cli-provided options
@@ -194,7 +207,7 @@ def validate_options(options: argparse.ArgumentParser):
 
 
 def create_authorized_workloads(
- SVID: JwtSvid, secret, server, port, users, groups, compute_nodes
+ SVID: JwtSvid, secret, url, users, groups, compute_nodes
 ):
  """Create workloads that are authorized to access to a secret
 
@@ -212,7 +225,7 @@ def create_authorized_workloads(
  """
 
  # Prepare request
- url = f"http://{server}:{port}/api/client/create-workloads"
+ url = f"{url}/api/client/create-workloads"
  payload = {
  "jwt": SVID.token,
  "secret": secret,
@@ -248,7 +261,9 @@ def create_authorized_workloads(
 
 if __name__ == "__main__":
  # Parse arguments from CLI
- options = parse_arguments()
+ options = parse_arguments() 
+ # Parse configuration file
+ configuration = parse_configuration(options.config)
 
  # Validate / Parse them
  (
@@ -277,15 +292,14 @@ def create_authorized_workloads(
  users_spiffeID, client_id, secrets_path, user_role = create_authorized_workloads(
  SVID,
  secret_name,
- options.sd_server_address,
- options.sd_server_port,
+ configuration["hpcs-server"]["url"],
  users,
  groups,
  compute_nodes,
  )
 
  # Login to the vault using client's certificate
- hvac_client = vault_login(SVID, f"client_{client_id}")
+ hvac_client = vault_login(configuration["vault"]["url"], SVID, f"client_{client_id}")
 
  # Prepare secret
  secret = {}
@@ -329,11 +343,11 @@ def create_authorized_workloads(
  ssh_copy_file(
  ssh_client,
  "/tmp/dataset_info.yaml",
- f"{options.data_path_at_rest}{secret_name}.info.yaml",
+ f"{options.data_path_at_rest}/{secret_name}.info.yaml",
  )
 
  print(
- f"Data and info file were shipped to te supercomputer. Infos about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml"
+ f"Data and info file were shipped to te supercomputer. Info about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml"
  )
 
  ssh_client.close()

utils/ssh_utils.py

@@ -1,5 +1,5 @@
 from paramiko.client import SSHClient
-from paramiko import SSHException, AutoAddPolicy
+from paramiko import SSHException, AutoAddPolicy, RSAKey
 from scp import SCPClient
 
 # Hostname and port configuration
@@ -30,11 +30,13 @@ def ssh_connect(username: str) -> SSHClient:
 
  # Probably running in a container
  except SSHException:
+ pkey=RSAKey.from_private_key_file("/tmp/.ssh/id_rsa")
  client.connect(
  host,
  port,
  username=username,
- key_filename="/tmp/.ssh/id_rsa",
+ pkey=pkey,
+ look_for_keys=False,
  auth_timeout=30,
  timeout=30,
  )