From b285a683bd3f836f3e42bcd73e4f5d59bdf5f0d2 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Mon, 28 Dec 2020 14:38:42 -0600 Subject: [PATCH 01/21] Legal Hold work to meet Issue 176 --- CHANGELOG.md | 3 + setup.py | 3 +- src/code42cli/cmds/legal_hold.py | 87 ++++++- tests/cmds/test_legal_hold.py | 399 +++++++++++++++++++++++++++++++ 4 files changed, 482 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bd7c4c5b..6e75bbfe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - `code42 audit-logs` commands: - `search` to search for audit-logs. - `send-to` to send audit-logs to server. + +- `code42 legal-hold show` option: + - `--include-devices` to print list of devices associated with legal hold storage along with storage by organization. ### Changed diff --git a/setup.py b/setup.py index afce8924b..c7329e316 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,9 @@ "colorama>=0.4.3", "c42eventextractor==0.4.0", "keyring==18.0.1", + "pandas>=1.1.3", "keyrings.alt==3.2.0", - "py42>=1.9", + "py42>=1.10", ], extras_require={ "dev": [ diff --git a/src/code42cli/cmds/legal_hold.py b/src/code42cli/cmds/legal_hold.py index 76c4bbc76..8e4c4c0f2 100644 --- a/src/code42cli/cmds/legal_hold.py +++ b/src/code42cli/cmds/legal_hold.py @@ -5,6 +5,7 @@ import click from click import echo +from pandas import DataFrame from code42cli.bulk import generate_template_cmd_factory from code42cli.bulk import run_bulk_process @@ -92,8 +93,19 @@ def _list(state, format=None): is_flag=True, help="View details of the preservation policy associated with the legal hold matter.", ) +@click.option( + "--include-devices", + is_flag=True, + help="View devices and storage associated with legal hold custodians.", +) @sdk_options() -def show(state, matter_id, include_inactive=False, include_policy=False): +def show( + state, + matter_id, + include_inactive=False, + include_policy=False, + include_devices=False, +): """Display details of a given legal hold matter.""" matter = _check_matter_is_accessible(state.sdk, matter_id) matter["creator_username"] = matter["creator"]["username"] @@ -105,19 +117,32 @@ def show(state, matter_id, include_inactive=False, include_policy=False): memberships = _get_legal_hold_memberships_for_matter( state.sdk, matter_id, active=active ) - active_usernames = [ - member["user"]["username"] for member in memberships if member["active"] - ] - inactive_usernames = [ - member["user"]["username"] for member in memberships if not member["active"] - ] formatter = OutputFormatter(OutputFormat.TABLE, _MATTER_KEYS_MAP) formatter.echo_formatted_list([matter]) - _print_matter_members(active_usernames, member_type="active") + users = [ + [member["active"], member["user"]["userUid"], member["user"]["username"]] + for member in memberships + ] + + usernames = [user[2] for user in users if user[0] is True] + _print_matter_members(usernames, member_type="active") if include_inactive: - _print_matter_members(inactive_usernames, member_type="inactive") + usernames = [user[2] for user in users if user[0] is not True] + _print_matter_members(usernames, member_type="inactive") + + if include_devices: + user_dataframe = _build_user_dataframe(users) + devices_dataframe = _merge_matter_members_with_devices( + state.sdk, user_dataframe + ) + if len(devices_dataframe.index) > 0: + echo("\nMatter Members and Devices:\n") + click.echo(devices_dataframe.to_csv()) + echo(_print_storage_by_org(devices_dataframe)) + else: + echo("\nNo devices associated with matter.\n") if include_policy: _get_and_print_preservation_policy(state.sdk, matter["holdPolicyUid"]) @@ -238,6 +263,50 @@ def _print_matter_members(username_list, member_type="active"): echo("No {} matter members.\n".format(member_type)) +def _merge_matter_members_with_devices(sdk, user_dataframe): + devices_generator = sdk.devices.get_all(active="true", include_backup_usage=True) + device_list = _get_total_archive_bytes_per_device(devices_generator) + devices_dataframe = DataFrame.from_records( + device_list, + columns=[ + "userUid", + "guid", + "name", + "osHostname", + "status", + "alertStates", + "orgId", + "lastConnected", + "version", + "archiveBytes", + ], + ) + return user_dataframe.merge( + devices_dataframe, how="inner", on="userUid" + ).reset_index(drop=True) + + +def _build_user_dataframe(users): + user_dataframe = DataFrame.from_records( + users, columns=["activeMembership", "userUid", "username"] + ) + return user_dataframe + + +def _get_total_archive_bytes_per_device(devices_generator): + device_list = [device for page in devices_generator for device in page["computers"]] + for i in device_list: + archive_bytes = [archive["archiveBytes"] for archive in i["backupUsage"]] + i["archiveBytes"] = sum(archive_bytes) + return device_list + + +def _print_storage_by_org(devices_dataframe): + echo("\nLegal Hold Storage by Org\n") + devices_dataframe = devices_dataframe.filter(["orgId", "archiveBytes"]) + return devices_dataframe.groupby("orgId").sum() + + @lru_cache(maxsize=None) def _check_matter_is_accessible(sdk, matter_id): return sdk.legalhold.get_matter_by_uid(matter_id) diff --git a/tests/cmds/test_legal_hold.py b/tests/cmds/test_legal_hold.py index e8c8655bc..585c73426 100644 --- a/tests/cmds/test_legal_hold.py +++ b/tests/cmds/test_legal_hold.py @@ -1,11 +1,17 @@ import pytest +from pandas import DataFrame +from pandas import testing from py42.exceptions import Py42BadRequestError from py42.response import Py42Response from requests import HTTPError from requests import Response from code42cli import PRODUCT_NAME +from code42cli.cmds.legal_hold import _build_user_dataframe from code42cli.cmds.legal_hold import _check_matter_is_accessible +from code42cli.cmds.legal_hold import _get_total_archive_bytes_per_device +from code42cli.cmds.legal_hold import _merge_matter_members_with_devices +from code42cli.cmds.legal_hold import _print_storage_by_org from code42cli.main import cli @@ -19,6 +25,8 @@ INACTIVE_TEST_USER_ID = "54321" TEST_POLICY_UID = "66666" TEST_PRESERVATION_POLICY_UID = "1010101010" +ACTIVE_TEST_DEVICE_GUID = "853543498784654695" +INACTIVE_TEST_DEVICE_GUID = "987879465123464477" MATTER_RESPONSE = """ { "legalHoldUid": "88888", @@ -38,6 +46,266 @@ "holdPolicyUid": "66666" } """ +TEST_DEVICE_PAGE = { + "computers": [ + { + "computerId": 1111111, + "name": "shouldprint", + "osHostname": "UNKNOWN", + "guid": "853543498784654695", + "type": "COMPUTER", + "status": "Active, Deauthorized", + "active": "True", + "blocked": "False", + "alertState": 2, + "alertStates": ["CriticalConnectionAlert"], + "userId": "12345", + "userUid": "12345", + "orgId": 521084, + "orgUid": "926779929022980902", + "computerExtRef": None, + "notes": None, + "parentComputerId": None, + "parentComputerGuid": None, + "lastConnected": "2020-03-16T17:06:50.774Z", + "osName": "linux", + "osVersion": "4.15.0-45-generic", + "osArch": "amd64", + "address": "xxx.xxx.x.xx:4242", + "remoteAddress": "xxx.xx.xxx.xxx", + "javaVersion": "1.8.0_144", + "modelInfo": None, + "timeZone": "America/Los_Angeles", + "version": 1525200006700, + "productVersion": "7.0.0", + "buildVersion": 586, + "creationDate": "2020-03-16T16:20:00.871Z", + "modificationDate": "2020-09-03T13:32:02.383Z", + "loginDate": "2020-03-16T16:52:18.900Z", + "service": "CrashPlan", + "backupUsage": [ + { + "targetComputerParentId": "null", + "targetComputerParentGuid": "null", + "targetComputerGuid": "632540230984925185", + "targetComputerName": "PROe Cloud, US - West", + "targetComputerOsName": "null", + "targetComputerType": "SERVER", + "selectedFiles": 0, + "selectedBytes": 0, + "todoFiles": 0, + "todoBytes": 0, + "archiveBytes": 99056, + "billableBytes": 99056, + "sendRateAverage": 0, + "completionRateAverage": 0, + "lastBackup": "null", + "lastCompletedBackup": "null", + "lastConnected": "null", + "lastMaintenanceDate": "2020-12-08T14:38:56.565-06:00", + "lastCompactDate": "2020-12-08T14:38:56.549-06:00", + "modificationDate": "2020-12-23T10:02:53.738-06:00", + "creationDate": "2020-04-06T16:50:44.353-05:00", + "using": "true", + "alertState": 16, + "alertStates": ["CriticalBackupAlert"], + "percentComplete": 0.0, + "storePointId": 12537, + "storePointName": "erf-sea-3", + "serverId": 160025225, + "serverGuid": "946058956729596234", + "serverName": "erf-sea", + "serverHostName": "https://web-erf-sea.crashplan.com", + "isProvider": "false", + "archiveGuid": "948688240625098914", + "archiveFormat": "ARCHIVE_V1", + "activity": { + "connected": "false", + "backingUp": "false", + "restoring": "false", + "timeRemainingInMs": 0, + "remainingFiles": 0, + "remainingBytes": 0, + }, + }, + { + "targetComputerParentId": "null", + "targetComputerParentGuid": "null", + "targetComputerGuid": "43", + "targetComputerName": "PROe Cloud, US", + "targetComputerOsName": "null", + "targetComputerType": "SERVER", + "selectedFiles": 63775, + "selectedBytes": 2434109067, + "todoFiles": 0, + "todoBytes": 0, + "archiveBytes": 1199319510, + "billableBytes": 2434109067, + "sendRateAverage": 10528400, + "completionRateAverage": 265154800, + "lastBackup": "2019-12-16T17:01:31.749-06:00", + "lastCompletedBackup": "2019-12-16T17:01:31.749-06:00", + "lastConnected": "2019-12-16T17:04:12.818-06:00", + "lastMaintenanceDate": "2020-11-19T20:02:02.054-06:00", + "lastCompactDate": "2020-11-19T20:02:02.051-06:00", + "modificationDate": "2020-12-23T06:58:08.684-06:00", + "creationDate": "2019-11-25T16:16:38.692-06:00", + "using": "true", + "alertState": 16, + "alertStates": ["CriticalBackupAlert"], + "percentComplete": 100.0, + "storePointId": 9788, + "storePointName": "eda-iad-1", + "serverId": 160023585, + "serverGuid": "829383329462096515", + "serverName": "eda-iad", + "serverHostName": "https://web-eda-iad.crashplan.com", + "isProvider": "false", + "archiveGuid": "929857309487839252", + "archiveFormat": "ARCHIVE_V1", + "activity": { + "connected": "false", + "backingUp": "false", + "restoring": "false", + "timeRemainingInMs": 0, + "remainingFiles": 0, + "remainingBytes": 0, + }, + }, + ], + }, + { + "computerId": 2222222, + "name": "shouldNotPrint", + "osHostname": "UNKNOWN", + "guid": "987879465123464477", + "type": "COMPUTER", + "status": "Active", + "active": "True", + "blocked": "False", + "alertState": 0, + "alertStates": ["OK"], + "userId": "02345", + "userUid": "02345", + "orgId": 521084, + "orgUid": "926779929022980902", + "computerExtRef": None, + "notes": None, + "parentComputerId": None, + "parentComputerGuid": None, + "lastConnected": "2018-03-19T20:04:02.999Z", + "osName": "win", + "osVersion": "10.0.18362", + "osArch": "amd64", + "address": "xxx.x.xx.xxx:4242", + "remoteAddress": "xx.xx.xxx.xxx", + "javaVersion": "11.04", + "modelInfo": None, + "timeZone": "America/Chicago", + "version": 1525200006770, + "productVersion": "7.7.0", + "buildVersion": 833, + "creationDate": "2020-03-19T19:43:16.918Z", + "modificationDate": "2020-09-08T15:43:45.875Z", + "loginDate": "2020-03-19T20:03:45.360Z", + "service": "CrashPlan", + "backupUsage": [ + { + "targetComputerParentId": "null", + "targetComputerParentGuid": "null", + "targetComputerGuid": "632540230984925185", + "targetComputerName": "PROe Cloud, US - West", + "targetComputerOsName": "null", + "targetComputerType": "SERVER", + "selectedFiles": 0, + "selectedBytes": 0, + "todoFiles": 0, + "todoBytes": 0, + "archiveBytes": 99056, + "billableBytes": 99056, + "sendRateAverage": 0, + "completionRateAverage": 0, + "lastBackup": "null", + "lastCompletedBackup": "null", + "lastConnected": "null", + "lastMaintenanceDate": "2020-12-08T14:38:56.565-06:00", + "lastCompactDate": "2020-12-08T14:38:56.549-06:00", + "modificationDate": "2020-12-23T10:02:53.738-06:00", + "creationDate": "2020-04-06T16:50:44.353-05:00", + "using": "true", + "alertState": 16, + "alertStates": ["CriticalBackupAlert"], + "percentComplete": 0.0, + "storePointId": 12537, + "storePointName": "erf-sea-3", + "serverId": 160025225, + "serverGuid": "946058956729596234", + "serverName": "erf-sea", + "serverHostName": "https://web-erf-sea.crashplan.com", + "isProvider": "false", + "archiveGuid": "948688240625098914", + "archiveFormat": "ARCHIVE_V1", + "activity": { + "connected": "false", + "backingUp": "false", + "restoring": "false", + "timeRemainingInMs": 0, + "remainingFiles": 0, + "remainingBytes": 0, + }, + }, + { + "targetComputerParentId": "null", + "targetComputerParentGuid": "null", + "targetComputerGuid": "43", + "targetComputerName": "PROe Cloud, US", + "targetComputerOsName": "null", + "targetComputerType": "SERVER", + "selectedFiles": 63775, + "selectedBytes": 2434109067, + "todoFiles": 0, + "todoBytes": 0, + "archiveBytes": 1199319510, + "billableBytes": 2434109067, + "sendRateAverage": 10528400, + "completionRateAverage": 265154800, + "lastBackup": "2019-12-16T17:01:31.749-06:00", + "lastCompletedBackup": "2019-12-16T17:01:31.749-06:00", + "lastConnected": "2019-12-16T17:04:12.818-06:00", + "lastMaintenanceDate": "2020-11-19T20:02:02.054-06:00", + "lastCompactDate": "2020-11-19T20:02:02.051-06:00", + "modificationDate": "2020-12-23T06:58:08.684-06:00", + "creationDate": "2019-11-25T16:16:38.692-06:00", + "using": "true", + "alertState": 16, + "alertStates": ["CriticalBackupAlert"], + "percentComplete": 100.0, + "storePointId": 9788, + "storePointName": "eda-iad-1", + "serverId": 160023585, + "serverGuid": "829383329462096515", + "serverName": "eda-iad", + "serverHostName": "https://web-eda-iad.crashplan.com", + "isProvider": "false", + "archiveGuid": "929857309487839252", + "archiveFormat": "ARCHIVE_V1", + "activity": { + "connected": "false", + "backingUp": "false", + "restoring": "false", + "timeRemainingInMs": 0, + "remainingFiles": 0, + "remainingBytes": 0, + }, + }, + ], + }, + ] +} +USERS_LIST = [ + [True, "12345", "user@example.com"], + [False, "02345", "inactive@example.com"], +] POLICY_RESPONSE = """ { "legalHoldPolicyUid": "1010101010", @@ -191,6 +459,11 @@ def preservation_policy_response(mocker): return _create_py42_response(mocker, POLICY_RESPONSE) +@pytest.fixture +def devices_list_generator(mocker): + return [TEST_DEVICE_PAGE] + + @pytest.fixture def empty_legal_hold_memberships_response(mocker): return [_create_py42_response(mocker, EMPTY_CUSTODIANS_RESPONSE)] @@ -238,6 +511,11 @@ def check_matter_accessible_success(cli_state, matter_response): cli_state.sdk.legalhold.get_matter_by_uid.return_value = matter_response +@pytest.fixture +def get_all_devices_success(cli_state, devices_list_generator): + cli_state.sdk.devices.get_all.return_value = devices_list_generator + + @pytest.fixture def check_matter_accessible_failure(cli_state): cli_state.sdk.legalhold.get_matter_by_uid.side_effect = Py42BadRequestError( @@ -500,6 +778,127 @@ def test_show_matter_prints_no_active_members_when_no_active_membership_and_inac assert "No active matter members." in result.output +def test_show_matter_prints_devices_when_active_user_has_devices_if_device_flag_is_set( + runner, + cli_state, + active_legal_hold_memberships_response, + check_matter_accessible_success, + get_all_devices_success, +): + + cli_state.sdk.legalhold.get_all_matter_custodians.return_value = ( + active_legal_hold_memberships_response + ) + result = runner.invoke( + cli, ["legal-hold", "show", TEST_MATTER_ID, "--include-devices"], obj=cli_state + ) + + assert ACTIVE_TEST_DEVICE_GUID in result.output + assert INACTIVE_TEST_DEVICE_GUID not in result.output + + +def test_show_matter_prints_devices_when_inactive_user_has_devices_if_device_and_inactive_flag_is_set( + runner, + cli_state, + active_and_inactive_legal_hold_memberships_response, + check_matter_accessible_success, + get_all_devices_success, +): + cli_state.sdk.legalhold.get_all_matter_custodians.return_value = ( + active_and_inactive_legal_hold_memberships_response + ) + result = runner.invoke( + cli, + [ + "legal-hold", + "show", + TEST_MATTER_ID, + "--include-devices", + "--include-inactive", + ], + obj=cli_state, + ) + + assert ACTIVE_TEST_DEVICE_GUID in result.output + assert INACTIVE_TEST_DEVICE_GUID in result.output + + +def test_show_matter_prints_no_device_table_if_no_devices_found_when_include_devices_flag_set( + runner, + cli_state, + active_and_inactive_legal_hold_memberships_response, + check_matter_accessible_success, +): + cli_state.sdk.legalhold.get_all_matter_custodians.return_value = ( + active_and_inactive_legal_hold_memberships_response + ) + cli_state.sdk.devices.get_all.return_value = {} + result = runner.invoke( + cli, + [ + "legal-hold", + "show", + TEST_MATTER_ID, + "--include-devices", + "--include-inactive", + ], + obj=cli_state, + ) + + assert "No devices associated with matter." in result.output + assert "Matter Members and Devices:" not in result.output + assert "Legal Hold Storage by Org" not in result.output + assert "osHostname" not in result.output + assert "alertStates" not in result.output + + +def test_show_matter_device_dataframe_returns_correct_columns(cli_state): + user_dataframe = _build_user_dataframe(USERS_LIST) + result = _merge_matter_members_with_devices(cli_state.sdk, user_dataframe) + assert "userUid" in result.columns + assert "username" in result.columns + assert "activeMembership" in result.columns + assert "guid" in result.columns + assert "lastConnected" in result.columns + assert "archiveBytes" in result.columns + assert "backupUsage" not in result.columns + + +def test_show_matter_user_dataframe_returns_correct_columns_and_values(): + result = _build_user_dataframe(USERS_LIST) + assert "userUid" in result.columns + assert "username" in result.columns + assert "activeMembership" in result.columns + assert "guid" not in result.columns + assert "12345" in result.values + assert "user@example.com" in result.values + + +def test_show_matter_org_storage_dataframe_returns_correct_group_values(): + test_devices_dataframe = DataFrame( + data={ + "orgId": [521084, 521084], + "guid": ["853543498784654695", "926779929022980902"], + "version": [1525200006770, 1525200006770], + "archiveBytes": [1199418566, 1199418566], + } + ) + expected_return = DataFrame.from_records( + [{"orgId": 521084, "archiveBytes": 2398837132}], index="orgId" + ) + test_return = _print_storage_by_org(test_devices_dataframe) + testing.assert_frame_equal(expected_return, test_return) + + +def test_show_matter_device_total_archive_bytes_are_calculated(devices_list_generator): + result = _get_total_archive_bytes_per_device(devices_list_generator) + assert "archiveBytes" in result[0].keys() + assert result[0]["archiveBytes"] == ( + result[0]["backupUsage"][0]["archiveBytes"] + + result[0]["backupUsage"][1]["archiveBytes"] + ) + + def test_show_matter_prints_preservation_policy_when_include_policy_flag_set( runner, cli_state, check_matter_accessible_success, preservation_policy_response ): From ed72879247130be031fa8e6e3dff737c2b937c41 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Mon, 28 Dec 2020 14:54:30 -0600 Subject: [PATCH 02/21] Fix to Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e75bbfe1..833ebe8a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - `code42 audit-logs` commands: - `search` to search for audit-logs. - `send-to` to send audit-logs to server. - + - `code42 legal-hold show` option: - `--include-devices` to print list of devices associated with legal hold storage along with storage by organization. From ea4381ad0809646101b408948f0920653fa6dba4 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 29 Dec 2020 09:09:53 -0600 Subject: [PATCH 03/21] Minor fix to CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 833ebe8a9..07a50a8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - `send-to` to send audit-logs to server. - `code42 legal-hold show` option: - - `--include-devices` to print list of devices associated with legal hold storage along with storage by organization. + - `--include-devices` to print list of devices associated with legal hold along with total storage by organization. ### Changed From a0ebe5702adfcb0d09e0d680173ff44b21bbbfd8 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 29 Dec 2020 09:18:53 -0600 Subject: [PATCH 04/21] Added to legal hold user guide --- docs/userguides/legalhold.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/userguides/legalhold.md b/docs/userguides/legalhold.md index a3d762be1..49db4fb0b 100644 --- a/docs/userguides/legalhold.md +++ b/docs/userguides/legalhold.md @@ -93,4 +93,8 @@ To view all custodians (including inactive) for a legal hold matter, enter `code42 legal-hold show --include-inactive` +To view all devices associated with selected custodians for a legal hold matter, enter + +`code42 legal-hold show --include-devices` + Learn more about the [Legal Hold](../commands/legalhold.md) commands. From 077dea1433fcbd0794f796d343919d46e4388999 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 29 Dec 2020 10:48:06 -0600 Subject: [PATCH 05/21] Adjusting build parameters to bypass 3.5 for this PR --- .github/workflows/build.yml | 2 +- setup.py | 3 +-- tox.ini | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8af91364a..104d59fa8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8] + python: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index c7329e316..efdcadea4 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ package_dir={"": "src"}, include_package_data=True, zip_safe=False, - python_requires=">3, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", + python_requires=">3, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4", install_requires=[ "click>=7.1.1", "colorama>=0.4.3", @@ -54,7 +54,6 @@ "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/tox.ini b/tox.ini index 6031d7c65..fc2236dff 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38,37,36,35} + py{38,37,36} docs style skip_missing_interpreters = true @@ -10,6 +10,7 @@ deps = pytest == 4.6.11 pytest-mock == 2.0.0 pytest-cov == 2.10.0 + pandas == 1.1.3 commands = # -v: verbose From f17499284191dfd865867249680e51d30fcd2e8d Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Mon, 4 Jan 2021 16:16:07 -0600 Subject: [PATCH 06/21] Fix low hanging fruit for initial PR review --- src/code42cli/cmds/legal_hold.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/code42cli/cmds/legal_hold.py b/src/code42cli/cmds/legal_hold.py index 8e4c4c0f2..74fd249fd 100644 --- a/src/code42cli/cmds/legal_hold.py +++ b/src/code42cli/cmds/legal_hold.py @@ -137,7 +137,7 @@ def show( devices_dataframe = _merge_matter_members_with_devices( state.sdk, user_dataframe ) - if len(devices_dataframe.index) > 0: + if len(devices_dataframe.index): echo("\nMatter Members and Devices:\n") click.echo(devices_dataframe.to_csv()) echo(_print_storage_by_org(devices_dataframe)) @@ -264,7 +264,7 @@ def _print_matter_members(username_list, member_type="active"): def _merge_matter_members_with_devices(sdk, user_dataframe): - devices_generator = sdk.devices.get_all(active="true", include_backup_usage=True) + devices_generator = sdk.devices.get_all(active=True, include_backup_usage=True) device_list = _get_total_archive_bytes_per_device(devices_generator) devices_dataframe = DataFrame.from_records( device_list, @@ -295,13 +295,13 @@ def _build_user_dataframe(users): def _get_total_archive_bytes_per_device(devices_generator): device_list = [device for page in devices_generator for device in page["computers"]] - for i in device_list: - archive_bytes = [archive["archiveBytes"] for archive in i["backupUsage"]] - i["archiveBytes"] = sum(archive_bytes) + for device in device_list: + archive_bytes = [archive["archiveBytes"] for archive in device["backupUsage"]] + device["archiveBytes"] = sum(archive_bytes) return device_list -def _print_storage_by_org(devices_dataframe): +def _get_storage_by_org(devices_dataframe): echo("\nLegal Hold Storage by Org\n") devices_dataframe = devices_dataframe.filter(["orgId", "archiveBytes"]) return devices_dataframe.groupby("orgId").sum() From 02d65302bd6aa49f2ed15f3d2a15098280078566 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 2 Feb 2021 09:29:52 -0600 Subject: [PATCH 07/21] remove whitespaces that are coming through as edits --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index ea3e751c6..83d505747 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,6 @@ deps = pandas == 1.1.3 pexpect == 4.8.0 - commands = # -v: verbose # -rsxX: show extra test summary info for (s)skipped, (x)failed, (X)passed From e5784cfe79ef844b10a40d552355d870c50bcf67 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 2 Feb 2021 10:13:18 -0600 Subject: [PATCH 08/21] fix changes identfied by tox style run --- CHANGELOG.md | 2 +- src/code42cli/cmds/legal_hold.py | 4 ++-- tests/cmds/test_devices.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50ec06585..0938460ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,7 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists. - Before, it would error and the cloud alias would not get added. - + ### Added - `code42 devices list` option: diff --git a/src/code42cli/cmds/legal_hold.py b/src/code42cli/cmds/legal_hold.py index 4529e1c87..76c4bbc76 100644 --- a/src/code42cli/cmds/legal_hold.py +++ b/src/code42cli/cmds/legal_hold.py @@ -111,11 +111,11 @@ def show(state, matter_id, include_inactive=False, include_policy=False): inactive_usernames = [ member["user"]["username"] for member in memberships if not member["active"] ] - + formatter = OutputFormatter(OutputFormat.TABLE, _MATTER_KEYS_MAP) formatter.echo_formatted_list([matter]) _print_matter_members(active_usernames, member_type="active") - + if include_inactive: _print_matter_members(inactive_usernames, member_type="inactive") diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 4dfdec94b..b4cd2ebd8 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -1,7 +1,7 @@ from datetime import date -import pytest import numpy as np +import pytest from pandas import DataFrame from pandas._testing import assert_frame_equal from py42.exceptions import Py42BadRequestError @@ -295,6 +295,7 @@ ] } + def _create_py42_response(mocker, text): response = mocker.MagicMock(spec=Response) response.text = text @@ -447,6 +448,7 @@ def get_all_custodian_success(cli_state): custodian_list_generator() ) + def test_deactivate_deactivates_device( runner, cli_state, deactivate_device_success, get_device_by_guid_success ): From 9bfcd4271b480bf2f0db1d04e6431fd5d4ec7789 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 2 Feb 2021 10:16:08 -0600 Subject: [PATCH 09/21] remove duplication in setup.py - file should have no edits --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f56777e83..722fbdecd 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ "keyring==18.0.1", "pandas>=1.1.3", "keyrings.alt==3.2.0", - "pandas>=1.1.3", "py42>=1.11", ], extras_require={ From 2e303de6ac711a843efc17c7189107947e920f50 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 2 Feb 2021 10:17:05 -0600 Subject: [PATCH 10/21] remove duplication in setup.py - file should have no edits --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 722fbdecd..68a782df4 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,8 @@ "colorama>=0.4.3", "c42eventextractor==0.4.0", "keyring==18.0.1", - "pandas>=1.1.3", "keyrings.alt==3.2.0", + "pandas>=1.1.3", "py42>=1.11", ], extras_require={ From 94fe3b96df95078c6fb1edc9f4b8fc65dc2caa02 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Thu, 11 Feb 2021 11:40:47 -0600 Subject: [PATCH 11/21] refactor membership function to use generator and remove NaNs from output --- src/code42cli/cmds/devices.py | 60 +++++++++++++---------------------- tests/cmds/test_devices.py | 16 ++++------ 2 files changed, 29 insertions(+), 47 deletions(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index d587cb72f..8ee2ce80b 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -4,6 +4,7 @@ import numpy as np from pandas import concat from pandas import DataFrame +from pandas import json_normalize from pandas import to_datetime from py42 import exceptions from py42.exceptions import Py42NotFoundError @@ -13,8 +14,6 @@ from code42cli.click_ext.groups import OrderedGroup from code42cli.click_ext.options import incompatible_with from code42cli.click_ext.types import MagicDate -from code42cli.cmds.legal_hold import _get_all_active_matters -from code42cli.cmds.legal_hold import _get_legal_hold_memberships_for_matter from code42cli.date_helper import round_datetime_to_day_end from code42cli.date_helper import round_datetime_to_day_start from code42cli.errors import Code42CLIError @@ -352,51 +351,36 @@ def list_devices( def _add_legal_hold_membership_to_device_dataframe(sdk, df): - matters = _get_all_active_matters(sdk) - matters = [matter["legalHoldUid"] for matter in matters] - - legal_hold_members = [] - - for matter in matters: - memberships = _get_legal_hold_memberships_for_matter( - sdk, matter_id=matter, active=None - ) - - legal_hold_members.extend( - [ - [ - str(member["active"]), - member["user"]["userUid"], - member["legalHold"]["legalHoldUid"], - member["legalHold"]["name"], - ] - for member in memberships - ] - ) + columns = ["legalHold.legalHoldUid", "legalHold.name", "user.userUid"] legal_hold_member_dataframe = ( - DataFrame.from_records( - legal_hold_members, - columns=[ - "legalHoldMemberActive", - "userUid", - "legalHoldUid", - "legalHoldName", - ], - ) - .groupby(["userUid"]) - .agg(lambda col: ",".join(col)) + json_normalize(list(_get_all_active_hold_memberships(sdk)))[columns] + .groupby(["user.userUid"]) + .agg(",".join) ) - df = df.merge(legal_hold_member_dataframe, how="left", on="userUid") + df = df.merge( + legal_hold_member_dataframe, + how="left", + left_on="userUid", + right_on="user.userUid", + ).fillna(value='') df.loc[ - df["status"] == "Deactivated", - ["legalHoldMemberActive", "legalHoldUid", "legalHoldName"], - ] = np.nan + df["status"] == "Deactivated", ["legalHold.legalHoldUid", "legalHold.name"], + ] = "" return df +def _get_all_active_hold_memberships(sdk): + for page in sdk.legalhold.get_all_matters(active=True): + for matter in page["legalHolds"]: + for _page in sdk.legalhold.get_all_matter_custodians( + legal_hold_uid=matter["legalHoldUid"], active=True + ): + yield from _page["legalHoldMemberships"] + + def _get_device_dataframe( sdk, columns, active=None, org_uid=None, include_backup_usage=False ): diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index b4cd2ebd8..325f970cb 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -661,9 +661,8 @@ def test_add_legal_hold_membership_to_device_dataframe_adds_legal_hold_columns_t ] ) result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf) - assert "legalHoldUid" in result.columns - assert "legalHoldMemberActive" in result.columns - assert "legalHoldName" in result.columns + assert "legalHold.legalHoldUid" in result.columns + assert "legalHold.name" in result.columns def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivated( @@ -681,20 +680,19 @@ def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivate { "userUid": "840103986007089121", "status": "Deactivated", - "legalHoldMemberActive": np.nan, - "legalHoldUid": np.nan, - "legalHoldName": np.nan, + "legalHold.legalHoldUid": "", + "legalHold.name": "", }, { "userUid": "840103986007089121", "status": "Active", - "legalHoldMemberActive": "True,True", - "legalHoldUid": "123456789,987654321", - "legalHoldName": "Test legal hold matter,Another Matter", + "legalHold.legalHoldUid": "123456789,987654321", + "legalHold.name": "Test legal hold matter,Another Matter", }, ] ) result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf) + assert_frame_equal(result, testdf_result) From 0b79911bd1d86a857b958a1c79e7a591ae596615 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Thu, 11 Feb 2021 11:43:39 -0600 Subject: [PATCH 12/21] fix tox style run issue --- src/code42cli/cmds/devices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 8ee2ce80b..3b46a2119 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -363,7 +363,7 @@ def _add_legal_hold_membership_to_device_dataframe(sdk, df): how="left", left_on="userUid", right_on="user.userUid", - ).fillna(value='') + ).fillna(value="") df.loc[ df["status"] == "Deactivated", ["legalHold.legalHoldUid", "legalHold.name"], From 1bd1e2f4edae149e00cba47c0c597b9d4335de81 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Thu, 11 Feb 2021 11:45:52 -0600 Subject: [PATCH 13/21] Fix tox style run x2 --- src/code42cli/cmds/devices.py | 1 - tests/cmds/test_devices.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 3b46a2119..928f76004 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -1,7 +1,6 @@ from datetime import date import click -import numpy as np from pandas import concat from pandas import DataFrame from pandas import json_normalize diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 325f970cb..98830b66e 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -1,6 +1,5 @@ from datetime import date -import numpy as np import pytest from pandas import DataFrame from pandas._testing import assert_frame_equal From e4725c72b1552ba68b498cbf2b3b999bc1a3d205 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 16 Feb 2021 16:14:04 -0600 Subject: [PATCH 14/21] flipping back to using NaN, awaiting PR #245 --- src/code42cli/cmds/devices.py | 5 +++-- tests/cmds/test_devices.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 928f76004..3cbefe408 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -1,6 +1,7 @@ from datetime import date import click +import numpy as np from pandas import concat from pandas import DataFrame from pandas import json_normalize @@ -362,11 +363,11 @@ def _add_legal_hold_membership_to_device_dataframe(sdk, df): how="left", left_on="userUid", right_on="user.userUid", - ).fillna(value="") + ) df.loc[ df["status"] == "Deactivated", ["legalHold.legalHoldUid", "legalHold.name"], - ] = "" + ] = np.nan return df diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 98830b66e..7dbd38a57 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -1,5 +1,6 @@ from datetime import date +import numpy as np import pytest from pandas import DataFrame from pandas._testing import assert_frame_equal @@ -679,8 +680,8 @@ def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivate { "userUid": "840103986007089121", "status": "Deactivated", - "legalHold.legalHoldUid": "", - "legalHold.name": "", + "legalHold.legalHoldUid": np.nan, + "legalHold.name": np.nan, }, { "userUid": "840103986007089121", From 237ea3102823fc59aa5f6370555e010f9a09cfb0 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Thu, 18 Feb 2021 11:06:46 -0600 Subject: [PATCH 15/21] Adding --include-total-storage option, which calculates total number of archives and archive bytes --- CHANGELOG.md | 3 ++- src/code42cli/cmds/devices.py | 37 ++++++++++++++++++++++++++++++++++- tests/cmds/test_devices.py | 31 ++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0938460ad..e9d5b72be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,8 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta ### Added - `code42 devices list` option: - - `--include-legal-hold-membership` to add legal hold columns to the result output, including legal hold matter name, UID, and whether the membership is active. + - `--include-legal-hold-membership` to add legal hold columns to the result output, printing the legal hold matter name and ID for any active device actively on legal hold + - `--include-total-storage` to add columns with total number of archives and total sum of archive bytes to each device in the result output ## 1.0.0 - 2020-08-31 diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 3cbefe408..fc997c621 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -5,6 +5,7 @@ from pandas import concat from pandas import DataFrame from pandas import json_normalize +from pandas import Series from pandas import to_datetime from py42 import exceptions from py42.exceptions import Py42NotFoundError @@ -254,6 +255,14 @@ def _get_device_info(sdk, device_guid): is_flag=True, help="Include legal hold membership in output.", ) +@click.option( + "--include-total-storage", + required=False, + type=bool, + default=False, + is_flag=True, + help="Include archive count and total storage in output.", +) @click.option( "--exclude-most-recently-connected", type=int, @@ -296,6 +305,7 @@ def list_devices( include_usernames, include_settings, include_legal_hold_membership, + include_total_storage, exclude_most_recently_connected, last_connected_after, last_connected_before, @@ -320,7 +330,11 @@ def list_devices( "userUid", ] df = _get_device_dataframe( - state.sdk, columns, active, org_uid, include_backup_usage + state.sdk, + columns, + active, + org_uid, + (include_backup_usage or include_total_storage), ) if last_connected_after: df = df.loc[to_datetime(df.lastConnected) > last_connected_after] @@ -337,6 +351,8 @@ def list_devices( .head(exclude_most_recently_connected) ) df = df.drop(most_recent.index) + if include_total_storage: + df = _add_storage_totals_to_dataframe(df, include_backup_usage) if include_settings: df = _add_settings_to_dataframe(state.sdk, df) if include_usernames: @@ -436,6 +452,25 @@ def _add_usernames_to_device_dataframe(sdk, device_dataframe): return device_dataframe.merge(users_dataframe, how="left", on="userUid") +def _add_storage_totals_to_dataframe(df, include_backup_usage): + df[["archiveCount", "totalStorageBytes"]] = df["backupUsage"].apply( + _break_backup_usage_into_total_storage + ) + + if not include_backup_usage: + df = df.drop("backupUsage", axis=1) + return df + + +def _break_backup_usage_into_total_storage(backup_usage): + total_storage = 0 + archive_count = 0 + for archive in backup_usage: + archive_count += 1 + total_storage += archive["archiveBytes"] + return Series([archive_count, total_storage]) + + @devices.command() @active_option @inactive_option diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 7dbd38a57..11d13b8a6 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -1,9 +1,12 @@ +import json from datetime import date import numpy as np import pytest from pandas import DataFrame +from pandas import Series from pandas._testing import assert_frame_equal +from pandas._testing import assert_series_equal from py42.exceptions import Py42BadRequestError from py42.exceptions import Py42ForbiddenError from py42.exceptions import Py42NotFoundError @@ -15,6 +18,7 @@ from code42cli.cmds.devices import _add_backup_set_settings_to_dataframe from code42cli.cmds.devices import _add_legal_hold_membership_to_device_dataframe from code42cli.cmds.devices import _add_usernames_to_device_dataframe +from code42cli.cmds.devices import _break_backup_usage_into_total_storage from code42cli.cmds.devices import _get_device_dataframe from code42cli.main import cli @@ -80,7 +84,19 @@ "836476656572622471","serverName":"cif-sea","serverHostName":"https://cif-sea.crashplan.com", "isProvider":false,"archiveGuid":"843293524842941560","archiveFormat":"ARCHIVE_V1","activity": {"connected":false,"backingUp":false,"restoring":false,"timeRemainingInMs":0, -"remainingFiles":0,"remainingBytes":0}}]}}""" +"remainingFiles":0,"remainingBytes":0}},{"targetComputerParentId":null,"targetComputerParentGuid": +null,"targetComputerGuid":"43","targetComputerName":"PROe Cloud, US","targetComputerOsName":null, +"targetComputerType":"SERVER","selectedFiles":1599,"selectedBytes":1529420143,"todoFiles":0, +"todoBytes":0,"archiveBytes":56848550,"billableBytes":1529420143,"sendRateAverage":0, +"completionRateAverage":0,"lastBackup":"2019-12-02T09:34:28.364-06:00","lastCompletedBackup": +"2019-12-02T09:34:28.364-06:00","lastConnected":"2019-12-02T11:02:36.108-06:00","lastMaintenanceDate": +"2021-02-16T07:01:11.697-06:00","lastCompactDate":"2021-02-16T07:01:11.694-06:00","modificationDate": +"2021-02-17T04:57:27.222-06:00","creationDate":"2019-09-26T15:27:38.806-05:00","using":true, +"alertState":16,"alertStates":["CriticalBackupAlert"],"percentComplete":100.0,"storePointId":10989, +"storePointName":"fsa-iad-2","serverId":160024121,"serverGuid":"883282371081742804","serverName": +"fsa-iad","serverHostName":"https://web-fsa-iad.crashplan.com","isProvider":false,"archiveGuid": +"92077743916530001","archiveFormat":"ARCHIVE_V1","activity":{"connected":false,"backingUp":false, +"restoring":false,"timeRemainingInMs":0,"remainingFiles":0,"remainingBytes":0}}]}}""" TEST_EMPTY_BACKUPUSAGE_RESPONSE = """{"metadata":{"timestamp":"2020-10-13T12:51:28.410Z","params": {"incBackupUsage":"True","idType":"guid"}},"data":{"computerId":1767,"name":"SNWINTEST1", "osHostname":"UNKNOWN","guid":"843290890230648046","type":"COMPUTER","status":"Active", @@ -711,6 +727,19 @@ def test_list_include_legal_hold_membership_merges_in_and_concats_legal_hold_inf assert "123456789,987654321" in result.output +def test_break_backup_usage_into_total_storage_correctly_calculates_values(): + test_backupusage_cell = json.loads(TEST_BACKUPUSAGE_RESPONSE)["data"]["backupUsage"] + result = _break_backup_usage_into_total_storage(test_backupusage_cell) + + test_empty_backupusage_cell = json.loads(TEST_EMPTY_BACKUPUSAGE_RESPONSE)["data"][ + "backupUsage" + ] + empty_result = _break_backup_usage_into_total_storage(test_empty_backupusage_cell) + + assert_series_equal(result, Series([2, 56968051])) + assert_series_equal(empty_result, Series([0, 0])) + + def test_last_connected_after_filters_appropriate_results( cli_state, runner, get_all_devices_success ): From 7212fbf5350d9ac786c223b95b23db07fea970a2 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Mon, 22 Feb 2021 15:21:27 -0600 Subject: [PATCH 16/21] Remove V2 archives from storage calcuation; rename columns --- src/code42cli/cmds/devices.py | 18 ++++++++++++------ tests/cmds/test_devices.py | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index fc997c621..6102e7774 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -261,7 +261,7 @@ def _get_device_info(sdk, device_guid): type=bool, default=False, is_flag=True, - help="Include archive count and total storage in output.", + help="Include backup archive count and total storage in output.", ) @click.option( "--exclude-most-recently-connected", @@ -373,6 +373,13 @@ def _add_legal_hold_membership_to_device_dataframe(sdk, df): json_normalize(list(_get_all_active_hold_memberships(sdk)))[columns] .groupby(["user.userUid"]) .agg(",".join) + .rename( + { + "legalHold.legalHoldUid": "legalHoldUid", + "legalHold.name": "legalHoldName", + }, + axis=1, + ) ) df = df.merge( legal_hold_member_dataframe, @@ -381,9 +388,7 @@ def _add_legal_hold_membership_to_device_dataframe(sdk, df): right_on="user.userUid", ) - df.loc[ - df["status"] == "Deactivated", ["legalHold.legalHoldUid", "legalHold.name"], - ] = np.nan + df.loc[df["status"] == "Deactivated", ["legalHoldUid", "legalHoldName"]] = np.nan return df @@ -466,8 +471,9 @@ def _break_backup_usage_into_total_storage(backup_usage): total_storage = 0 archive_count = 0 for archive in backup_usage: - archive_count += 1 - total_storage += archive["archiveBytes"] + if archive["archiveFormat"] == "ARCHIVE_V1": + archive_count += 1 + total_storage += archive["archiveBytes"] return Series([archive_count, total_storage]) diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 11d13b8a6..5dbf31a22 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -677,8 +677,8 @@ def test_add_legal_hold_membership_to_device_dataframe_adds_legal_hold_columns_t ] ) result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf) - assert "legalHold.legalHoldUid" in result.columns - assert "legalHold.name" in result.columns + assert "legalHoldUid" in result.columns + assert "legalHoldName" in result.columns def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivated( @@ -696,14 +696,14 @@ def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivate { "userUid": "840103986007089121", "status": "Deactivated", - "legalHold.legalHoldUid": np.nan, - "legalHold.name": np.nan, + "legalHoldUid": np.nan, + "legalHoldName": np.nan, }, { "userUid": "840103986007089121", "status": "Active", - "legalHold.legalHoldUid": "123456789,987654321", - "legalHold.name": "Test legal hold matter,Another Matter", + "legalHoldUid": "123456789,987654321", + "legalHoldName": "Test legal hold matter,Another Matter", }, ] ) From 2a917bd5e8c4354f77d65ed7036f00653ca0253e Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Tue, 23 Feb 2021 09:43:41 -0600 Subject: [PATCH 17/21] fix small change to the incldue/excluded archive types --- src/code42cli/cmds/devices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 6102e7774..8935752b2 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -471,7 +471,7 @@ def _break_backup_usage_into_total_storage(backup_usage): total_storage = 0 archive_count = 0 for archive in backup_usage: - if archive["archiveFormat"] == "ARCHIVE_V1": + if archive["archiveFormat"] != "ARCHIVE_V2": archive_count += 1 total_storage += archive["archiveBytes"] return Series([archive_count, total_storage]) From 2d1db8cb55ac6c210b7e2ed69411af93defc0547 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Thu, 25 Feb 2021 14:09:34 -0600 Subject: [PATCH 18/21] reword --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d5b72be..4a902a733 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,7 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta ### Added - `code42 devices list` option: - - `--include-legal-hold-membership` to add legal hold columns to the result output, printing the legal hold matter name and ID for any active device actively on legal hold + - `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold - `--include-total-storage` to add columns with total number of archives and total sum of archive bytes to each device in the result output ## 1.0.0 - 2020-08-31 From 9a0afcbcc72a4d3ebe5b9123144f52045cb03e78 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Fri, 26 Feb 2021 11:13:41 -0600 Subject: [PATCH 19/21] conflict reconciliation in changelog, part I --- CHANGELOG.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a902a733..bfe53cad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,9 +58,6 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - `search` to search for audit-logs. - `send-to` to send audit-logs to server. - -## Unreleased - ### Changed - `profile_name` argument is now required for `code42 profile delete`, as it was meant to be. @@ -72,12 +69,6 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists. - Before, it would error and the cloud alias would not get added. -### Added - -- `code42 devices list` option: - - `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold - - `--include-total-storage` to add columns with total number of archives and total sum of archive bytes to each device in the result output - ## 1.0.0 - 2020-08-31 ### Fixed From a3dd28f5ca7da31403722651c5971536cb9f11d3 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Fri, 26 Feb 2021 11:34:15 -0600 Subject: [PATCH 20/21] conflict reconciliation in changelog, part II (repulled from upstream master --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe53cad0..ac0159291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The intended audience of this file is for py42 consumers -- as such, changes that don't affect how a consumer would use the library (e.g. adding unit tests, updating documentation, etc) are not captured here. +## 1.3.1 - 2021-02-25 + +### Changed + +- Command options for `profile update`: + - `-n` `--name` is not required, and if omitted will use the default profile. + - `-s` `--server` and `-u` `--username` are not required and can be updated independently now. + - Example: `code42 profile update -s 1.2.3.4:1234` + +## 1.3.0 - 2021-02-11 + +### Fixed + +- Issue where `code42 alert-rules bulk add` would show as successful when adding users to a non-existent alert rule. + +### Added + +- New choice `TLS-TCP` for `--protocol` option used by `send-to` commands: + - `code42 security-data send-to` + - `code42 alerts send-to` + - `code42 audit-logs send-to` + for more securely transporting data. Included are new flags: + - `--certs` + - `--ignore-cert-validation` + +### Changed + +- The error text in cases command when: + - `cases create` sets a name that already exists in the system. + - `cases create` sets a description that has more than 250 characters. + - `cases update` sets a description that has more than 250 characters. + - `cases file-events add` is performed on an already closed case. + - `cases file-events add` sets an event id that is already added to the case. + - `cases file-events remove` is performed on an already closed case. + ## 1.2.0 - 2021-01-25 ### Added @@ -58,6 +93,8 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - `search` to search for audit-logs. - `send-to` to send audit-logs to server. +## Unreleased + ### Changed - `profile_name` argument is now required for `code42 profile delete`, as it was meant to be. @@ -68,6 +105,12 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists. - Before, it would error and the cloud alias would not get added. + +### Added + +- `code42 devices list` option: + - `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold + - `--include-total-storage` prints the backup archive count and total storage ## 1.0.0 - 2020-08-31 From 9677f238910c46d686a5e364c374e19eefddd6c7 Mon Sep 17 00:00:00 2001 From: Maddie Vargo Date: Fri, 26 Feb 2021 11:47:17 -0600 Subject: [PATCH 21/21] fix style run --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0159291..a06ac9cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,12 +105,12 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists. - Before, it would error and the cloud alias would not get added. - + ### Added - `code42 devices list` option: - `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold - - `--include-total-storage` prints the backup archive count and total storage + - `--include-total-storage` prints the backup archive count and total storage ## 1.0.0 - 2020-08-31