From c08abb18a69f075d1074e48edefa5ed8bc081fbd Mon Sep 17 00:00:00 2001 From: Julie Rakas Date: Mon, 23 Sep 2024 17:07:10 +0000 Subject: [PATCH] efs-utils v2.1.0-1 release - Support region as a mount option (PR-171) - Add new regions to efs-utils.conf file (PR-241) --- .circleci/config.yml | 9 ++++++ README.md | 8 ++++- amazon-efs-utils.spec | 14 +++++---- build-deb.sh | 4 +-- config.ini | 4 +-- dist/amazon-efs-utils.control | 2 +- dist/efs-utils.conf | 16 ++++++++++ man/mount.efs.8 | 3 ++ requirements.txt | 30 +++++++------------ src/mount_efs/__init__.py | 19 +++++++----- src/proxy/Cargo.toml | 4 +-- src/watchdog/__init__.py | 2 +- .../test_get_target_instance_identity.py | 16 ++++++---- 13 files changed, 84 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04c22ea8..e7179b56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,6 +232,15 @@ jobs: workflows: workflow: jobs: + - test: + name: python3_12 + image: python:3.12.4 + - test: + name: python3_11 + image: python:3.11.9 + - test: + name: python3_10 + image: python:3.10.13 - test: name: python3_9 image: python:3.9.13 diff --git a/README.md b/README.md index c9ef9e63..dbe31e2e 100644 --- a/README.md +++ b/README.md @@ -260,12 +260,18 @@ To mount file system within a given network namespace, run: $ sudo mount -t efs -o netns=netns-path file-system-id efs-mount-point/ ``` -To mount file system to the mount target in specific availability zone (e.g. us-east-1a), run: +To mount file system to the mount target in a specific availability zone (e.g. us-east-1a), run: ```bash $ sudo mount -t efs -o az=az-name file-system-id efs-mount-point/ ``` +To mount file system to the mount target in a specific region (e.g. us-east-1), run: + +```bash +$ sudo mount -t efs -o region=region-name file-system-id efs-mount-point/ +``` + **Note: The [prequisites in the crossaccount section below](#crossaccount-option-prerequisites) must be completed before using the crossaccount option.** To mount the filesystem mount target in the same physical availability zone ID (e.g. use1-az1) as the client instance over cross-AWS-account mounts, run: diff --git a/amazon-efs-utils.spec b/amazon-efs-utils.spec index 68e96fe3..bbd16d3c 100644 --- a/amazon-efs-utils.spec +++ b/amazon-efs-utils.spec @@ -41,8 +41,8 @@ %{?!include_vendor_tarball:%define include_vendor_tarball true} Name : amazon-efs-utils -Version : 2.0.4 -Release : 2%{platform} +Version : 2.1.0 +Release : 1%{platform} Summary : This package provides utilities for simplifying the use of EFS file systems Group : Amazon/Tools @@ -192,22 +192,26 @@ fi %clean %changelog +* Wed Sep 18 2024 Julie Rakas - 2.1.0 +- Add mount option for specifying region +- Add new ISO regions to config file + * Tue Jun 25 2024 Anthony Tse - 2.0.4 - Add retry logic to and increase timeout for EC2 metadata token retrieval requests * Tue Jun 18 2024 Arnav Gupta - 2.0.3 - Upgrade py version -- Replace deprecated usage of datetime +- Replace deprecated usage of datetime * Mon May 20 2024 Anthony Tse - 2.0.2 - Check for efs-proxy PIDs when cleaning tunnel state files - Add PID to log entries * Tue Apr 23 2024 Ryan Stankiewicz - 2.0.1 -- Disable Nagle's algorithm for efs-proxy TLS mounts to improve latencies +- Disable Nagle's algorithm for efs-proxy TLS mounts to improve latencies * Mon Apr 08 2024 Ryan Stankiewicz - 2.0.0 -- Replace stunnel, which provides TLS encryptions for mounts, with efs-proxy, a component built in-house at AWS. Efs-proxy lays the foundation for upcoming feature launches at EFS. +- Replace stunnel, which provides TLS encryptions for mounts, with efs-proxy, a component built in-house at AWS. Efs-proxy lays the foundation for upcoming feature launches at EFS. * Mon Mar 18 2024 Sean Zatz - 1.36.0 - Support new mount option: crossaccount, conduct cross account mounts via ip address. Use client AZ-ID to choose mount target. diff --git a/build-deb.sh b/build-deb.sh index 278f2505..76be9aff 100755 --- a/build-deb.sh +++ b/build-deb.sh @@ -11,8 +11,8 @@ set -ex BASE_DIR=$(pwd) BUILD_ROOT=${BASE_DIR}/build/debbuild -VERSION=2.0.4 -RELEASE=2 +VERSION=2.1.0 +RELEASE=1 DEB_SYSTEM_RELEASE_PATH=/etc/os-release echo 'Cleaning deb build workspace' diff --git a/config.ini b/config.ini index a57dd738..86a460dd 100644 --- a/config.ini +++ b/config.ini @@ -7,5 +7,5 @@ # [global] -version=2.0.4 -release=2 +version=2.1.0 +release=1 diff --git a/dist/amazon-efs-utils.control b/dist/amazon-efs-utils.control index 7734f8e6..d3f7b0c0 100644 --- a/dist/amazon-efs-utils.control +++ b/dist/amazon-efs-utils.control @@ -1,6 +1,6 @@ Package: amazon-efs-utils Architecture: all -Version: 2.0.4 +Version: 2.1.0 Section: utils Depends: python3, nfs-common, stunnel4 (>= 4.56), openssl (>= 1.0.2), util-linux Priority: optional diff --git a/dist/efs-utils.conf b/dist/efs-utils.conf index 1b6d849e..5d9482fa 100644 --- a/dist/efs-utils.conf +++ b/dist/efs-utils.conf @@ -74,6 +74,22 @@ stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem dns_name_suffix = sc2s.sgov.gov stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +[mount.us-isob-west-1] +dns_name_suffix = sc2s.sgov.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.us-isof-east-1] +dns_name_suffix = csp.hci.ic.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.us-isof-south-1] +dns_name_suffix = csp.hci.ic.gov +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + +[mount.eu-isoe-west-1] +dns_name_suffix = cloud.adc-e.uk +stunnel_cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + [mount-watchdog] enabled = true poll_interval_sec = 1 diff --git a/man/mount.efs.8 b/man/mount.efs.8 index f962fd97..a3a807a7 100644 --- a/man/mount.efs.8 +++ b/man/mount.efs.8 @@ -79,6 +79,9 @@ this option is by default passed and the EFS file system is mounted over TLS\&. \fBnotls\fR Mounts the EFS file system without TLS, applies for Mac distributions only\&. .TP +\fBregion\fR +Mounts the EFS file system from the specified region, overriding any config file value\&. +.TP \fBtlsport=\fR\fIn\fR Configures the proxy process to listen for connections from the NFS client on the specified port\&. This is applicable to both non-tls and tls mounts. By default, the \ diff --git a/requirements.txt b/requirements.txt index 05dffd55..ccc3de50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,10 @@ -attrs==17.4.0 -botocore==1.17.53 -configparser==3.5.0 -coverage==4.5.4 -enum34==1.1.6 -flake8==3.7.9 -funcsigs==1.0.2 -mccabe==0.6.1 -mock==2.0.0 -pbr==3.1.1 -pluggy==0.13.0 -py==1.11.0 -pycodestyle==2.5.0 -pyflakes==2.1.1 -pytest==4.6.7 -pytest-cov==2.8.1 -pytest-html==1.19.0 -pytest-metadata==1.7.0 -pytest-mock==1.11.2 -six==1.11.0 +botocore == 1.34.140 +configparser == 7.0.0 +coverage == 7.6.0 +flake8 == 7.1.0 +pytest == 8.2.2 +pytest-cov == 5.0.0 +pytest-html == 4.1.1 +pytest-metadata == 3.1.1 +pytest-mock == 3.14.0 +mock == 5.1.0 diff --git a/src/mount_efs/__init__.py b/src/mount_efs/__init__.py index 083e3fa1..675a18ca 100755 --- a/src/mount_efs/__init__.py +++ b/src/mount_efs/__init__.py @@ -85,7 +85,7 @@ BOTOCORE_PRESENT = False -VERSION = "2.0.4" +VERSION = "2.1.0" SERVICE = "elasticfilesystem" AMAZON_LINUX_2_RELEASE_ID = "Amazon Linux release 2 (Karoo)" @@ -242,6 +242,7 @@ "noocsp", "notls", "ocsp", + "region", "tls", "tlsport", "verify", @@ -370,14 +371,18 @@ def fatal_error(user_message, log_message=None, exit_code=1): sys.exit(exit_code) -def get_target_region(config): +def get_target_region(config, options): def _fatal_error(message): fatal_error( 'Error retrieving region. Please set the "region" parameter ' - "in the efs-utils configuration file.", + "in the efs-utils configuration file or specify it as a " + "mount option.", message, ) + if "region" in options: + return options.get("region") + try: return config.get(CONFIG_SECTION, "region") except NoOptionError: @@ -1747,7 +1752,7 @@ def bootstrap_proxy( cert_details = None security_credentials = None client_info = get_client_info(config) - region = get_target_region(config) + region = get_target_region(config, options) if tls_enabled(options): cert_details = {} @@ -2662,7 +2667,7 @@ def _validate_replacement_field_count(format_str, expected_ct): if options and "crossaccount" in options: try: az_id = get_az_id_from_instance_metadata(config, options) - region = get_target_region(config) + region = get_target_region(config, options) dns_name = "%s.%s.efs.%s.amazonaws.com" % (az_id, fs_id, region) except RuntimeError: err_msg = "Cannot retrieve AZ-ID from metadata service. This is required for the crossaccount mount option." @@ -2687,7 +2692,7 @@ def _validate_replacement_field_count(format_str, expected_ct): if "{region}" in dns_name_format: expected_replacement_field_ct += 1 - format_args["region"] = get_target_region(config) + format_args["region"] = get_target_region(config, options) if "{dns_name_suffix}" in dns_name_format: expected_replacement_field_ct += 1 @@ -3380,7 +3385,7 @@ def get_botocore_client(config, service, options): botocore_config = botocore.config.Config(use_fips_endpoint=True) session = botocore.session.get_session() - region = get_target_region(config) + region = get_target_region(config, options) if options and options.get("awsprofile"): profile = options.get("awsprofile") diff --git a/src/proxy/Cargo.toml b/src/proxy/Cargo.toml index b5f1d721..602677c5 100644 --- a/src/proxy/Cargo.toml +++ b/src/proxy/Cargo.toml @@ -3,7 +3,7 @@ name = "efs-proxy" edition = "2021" build = "build.rs" # The version of efs-proxy is tied to efs-utils. -version = "2.0.4" +version = "2.1.0" publish = false [dependencies] @@ -25,7 +25,7 @@ s2n-tls-sys = "0.0" serde = {version="1.0.175",features=["derive"]} serde_ini = "0.2.0" thiserror = "1.0.44" -tokio = { version = "1.29.0", features = ["full"] } +tokio = { version = "1.29.0, <1.39", features = ["full"] } tokio-util = "0.7.8" uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]} xdr-codec = "0.4.4" diff --git a/src/watchdog/__init__.py b/src/watchdog/__init__.py index 3247a380..d9f620aa 100755 --- a/src/watchdog/__init__.py +++ b/src/watchdog/__init__.py @@ -56,7 +56,7 @@ AMAZON_LINUX_2_RELEASE_ID, AMAZON_LINUX_2_PRETTY_NAME, ] -VERSION = "2.0.4" +VERSION = "2.1.0" SERVICE = "elasticfilesystem" CONFIG_FILE = "/etc/amazon/efs/efs-utils.conf" diff --git a/test/mount_efs_test/test_get_target_instance_identity.py b/test/mount_efs_test/test_get_target_instance_identity.py index 6700fbea..e3672793 100644 --- a/test/mount_efs_test/test_get_target_instance_identity.py +++ b/test/mount_efs_test/test_get_target_instance_identity.py @@ -84,9 +84,9 @@ def get_config(dns_name_format, region=None): return config -def get_target_region_helper(): +def get_target_region_helper(options={}): config = get_config(DEFAULT_DNS_NAME_FORMAT) - return mount_efs.get_target_region(config) + return mount_efs.get_target_region(config, options) def get_target_az_helper(options={}): @@ -166,7 +166,7 @@ def test_get_target_region_from_metadata(mocker): mocker.patch("mount_efs.get_aws_ec2_metadata_token", return_value=None) mocker.patch("mount_efs.urlopen", return_value=MockUrlLibResponse()) config = get_config("{fs_id}.efs.{region}.{dns_name_suffix}", None) - assert TARGET_REGION == mount_efs.get_target_region(config) + assert TARGET_REGION == mount_efs.get_target_region(config, {}) def test_get_target_region_config_metadata_unavailable(mocker, capsys): @@ -174,7 +174,7 @@ def test_get_target_region_config_metadata_unavailable(mocker, capsys): mocker.patch("mount_efs.urlopen", side_effect=URLError("test error")) config = get_config("{fs_id}.efs.{region}.{dns_name_suffix}") with pytest.raises(SystemExit) as ex: - mount_efs.get_target_region(config) + mount_efs.get_target_region(config, {}) assert 0 != ex.value.code out, err = capsys.readouterr() @@ -232,13 +232,13 @@ def test_get_target_region_missing_region(mocker, capsys): def test_get_target_region_from_config_variable(mocker): config = get_config("{az}.{fs_id}.efs.us-east-2.{dns_name_suffix}", TARGET_REGION) - assert TARGET_REGION == mount_efs.get_target_region(config) + assert TARGET_REGION == mount_efs.get_target_region(config, {}) def _test_get_target_region_from_dns_format(mocker, config): mocker.patch("mount_efs.get_aws_ec2_metadata_token", return_value=None) mocker.patch("mount_efs.urlopen", side_effect=URLError("test error")) - assert TARGET_REGION == mount_efs.get_target_region(config) + assert TARGET_REGION == mount_efs.get_target_region(config, {}) def test_get_target_region_from_legacy_dns_name_format(mocker): @@ -277,3 +277,7 @@ def test_get_target_az_not_present_in_options_and_instance_metadata(mocker): def test_get_target_az_from_options(mocker): assert TARGET_AZ == get_target_az_helper(options={"az": TARGET_AZ}) + + +def test_get_target_region_from_options(mocker): + assert TARGET_REGION == get_target_region_helper(options={"region": TARGET_REGION})