Skip to content

Commit 65c5426

Browse files
maddie-vargomaddie-vargo
authored
Add legal hold membership to device reporting (#192)
* Legal Hold work to meet Issue 176 * Fix to Changelog * Minor fix to CHANGELOG * Added to legal hold user guide * Adjusting build parameters to bypass 3.5 for this PR * Fix low hanging fruit for initial PR review * remove whitespaces that are coming through as edits * fix changes identfied by tox style run * remove duplication in setup.py - file should have no edits * remove duplication in setup.py - file should have no edits * refactor membership function to use generator and remove NaNs from output * fix tox style run issue * Fix tox style run x2 * flipping back to using NaN, awaiting PR #245 * Adding --include-total-storage option, which calculates total number of archives and archive bytes * Remove V2 archives from storage calcuation; rename columns * fix small change to the incldue/excluded archive types * reword * conflict reconciliation in changelog, part I * conflict reconciliation in changelog, part II (repulled from upstream master * fix style run
1 parent adf3f44 commit 65c5426

File tree

3 files changed

+274
-2
lines changed

3 files changed

+274
-2
lines changed

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta
106106
- Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists.
107107
- Before, it would error and the cloud alias would not get added.
108108

109+
### Added
110+
111+
- `code42 devices list` option:
112+
- `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold
113+
- `--include-total-storage` prints the backup archive count and total storage
114+
109115
## 1.0.0 - 2020-08-31
110116

111117
### Fixed

‎src/code42cli/cmds/devices.py

+86-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
from datetime import date
22

33
import click
4+
import numpy as np
45
from pandas import concat
56
from pandas import DataFrame
7+
from pandas import json_normalize
8+
from pandas import Series
69
from pandas import to_datetime
710
from py42 import exceptions
811
from py42.exceptions import Py42NotFoundError
@@ -244,6 +247,22 @@ def _get_device_info(sdk, device_guid):
244247
is_flag=True,
245248
help="Include device settings in output.",
246249
)
250+
@click.option(
251+
"--include-legal-hold-membership",
252+
required=False,
253+
type=bool,
254+
default=False,
255+
is_flag=True,
256+
help="Include legal hold membership in output.",
257+
)
258+
@click.option(
259+
"--include-total-storage",
260+
required=False,
261+
type=bool,
262+
default=False,
263+
is_flag=True,
264+
help="Include backup archive count and total storage in output.",
265+
)
247266
@click.option(
248267
"--exclude-most-recently-connected",
249268
type=int,
@@ -285,6 +304,8 @@ def list_devices(
285304
include_backup_usage,
286305
include_usernames,
287306
include_settings,
307+
include_legal_hold_membership,
308+
include_total_storage,
288309
exclude_most_recently_connected,
289310
last_connected_after,
290311
last_connected_before,
@@ -309,7 +330,11 @@ def list_devices(
309330
"userUid",
310331
]
311332
df = _get_device_dataframe(
312-
state.sdk, columns, active, org_uid, include_backup_usage
333+
state.sdk,
334+
columns,
335+
active,
336+
org_uid,
337+
(include_backup_usage or include_total_storage),
313338
)
314339
if last_connected_after:
315340
df = df.loc[to_datetime(df.lastConnected) > last_connected_after]
@@ -326,17 +351,57 @@ def list_devices(
326351
.head(exclude_most_recently_connected)
327352
)
328353
df = df.drop(most_recent.index)
354+
if include_total_storage:
355+
df = _add_storage_totals_to_dataframe(df, include_backup_usage)
329356
if include_settings:
330357
df = _add_settings_to_dataframe(state.sdk, df)
331358
if include_usernames:
332359
df = _add_usernames_to_device_dataframe(state.sdk, df)
360+
if include_legal_hold_membership:
361+
df = _add_legal_hold_membership_to_device_dataframe(state.sdk, df)
333362
if df.empty:
334363
click.echo("No results found.")
335364
else:
336365
formatter = DataFrameOutputFormatter(format)
337366
formatter.echo_formatted_dataframe(df)
338367

339368

369+
def _add_legal_hold_membership_to_device_dataframe(sdk, df):
370+
columns = ["legalHold.legalHoldUid", "legalHold.name", "user.userUid"]
371+
372+
legal_hold_member_dataframe = (
373+
json_normalize(list(_get_all_active_hold_memberships(sdk)))[columns]
374+
.groupby(["user.userUid"])
375+
.agg(",".join)
376+
.rename(
377+
{
378+
"legalHold.legalHoldUid": "legalHoldUid",
379+
"legalHold.name": "legalHoldName",
380+
},
381+
axis=1,
382+
)
383+
)
384+
df = df.merge(
385+
legal_hold_member_dataframe,
386+
how="left",
387+
left_on="userUid",
388+
right_on="user.userUid",
389+
)
390+
391+
df.loc[df["status"] == "Deactivated", ["legalHoldUid", "legalHoldName"]] = np.nan
392+
393+
return df
394+
395+
396+
def _get_all_active_hold_memberships(sdk):
397+
for page in sdk.legalhold.get_all_matters(active=True):
398+
for matter in page["legalHolds"]:
399+
for _page in sdk.legalhold.get_all_matter_custodians(
400+
legal_hold_uid=matter["legalHoldUid"], active=True
401+
):
402+
yield from _page["legalHoldMemberships"]
403+
404+
340405
def _get_device_dataframe(
341406
sdk, columns, active=None, org_uid=None, include_backup_usage=False
342407
):
@@ -392,6 +457,26 @@ def _add_usernames_to_device_dataframe(sdk, device_dataframe):
392457
return device_dataframe.merge(users_dataframe, how="left", on="userUid")
393458

394459

460+
def _add_storage_totals_to_dataframe(df, include_backup_usage):
461+
df[["archiveCount", "totalStorageBytes"]] = df["backupUsage"].apply(
462+
_break_backup_usage_into_total_storage
463+
)
464+
465+
if not include_backup_usage:
466+
df = df.drop("backupUsage", axis=1)
467+
return df
468+
469+
470+
def _break_backup_usage_into_total_storage(backup_usage):
471+
total_storage = 0
472+
archive_count = 0
473+
for archive in backup_usage:
474+
if archive["archiveFormat"] != "ARCHIVE_V2":
475+
archive_count += 1
476+
total_storage += archive["archiveBytes"]
477+
return Series([archive_count, total_storage])
478+
479+
395480
@devices.command()
396481
@active_option
397482
@inactive_option

‎tests/cmds/test_devices.py

+182-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1+
import json
12
from datetime import date
23

4+
import numpy as np
35
import pytest
46
from pandas import DataFrame
7+
from pandas import Series
8+
from pandas._testing import assert_frame_equal
9+
from pandas._testing import assert_series_equal
510
from py42.exceptions import Py42BadRequestError
611
from py42.exceptions import Py42ForbiddenError
712
from py42.exceptions import Py42NotFoundError
@@ -10,7 +15,9 @@
1015

1116
from code42cli import PRODUCT_NAME
1217
from code42cli.cmds.devices import _add_backup_set_settings_to_dataframe
18+
from code42cli.cmds.devices import _add_legal_hold_membership_to_device_dataframe
1319
from code42cli.cmds.devices import _add_usernames_to_device_dataframe
20+
from code42cli.cmds.devices import _break_backup_usage_into_total_storage
1421
from code42cli.cmds.devices import _get_device_dataframe
1522
from code42cli.main import cli
1623

@@ -76,7 +83,19 @@
7683
"836476656572622471","serverName":"cif-sea","serverHostName":"https://cif-sea.crashplan.com",
7784
"isProvider":false,"archiveGuid":"843293524842941560","archiveFormat":"ARCHIVE_V1","activity":
7885
{"connected":false,"backingUp":false,"restoring":false,"timeRemainingInMs":0,
79-
"remainingFiles":0,"remainingBytes":0}}]}}"""
86+
"remainingFiles":0,"remainingBytes":0}},{"targetComputerParentId":null,"targetComputerParentGuid":
87+
null,"targetComputerGuid":"43","targetComputerName":"PROe Cloud, US","targetComputerOsName":null,
88+
"targetComputerType":"SERVER","selectedFiles":1599,"selectedBytes":1529420143,"todoFiles":0,
89+
"todoBytes":0,"archiveBytes":56848550,"billableBytes":1529420143,"sendRateAverage":0,
90+
"completionRateAverage":0,"lastBackup":"2019-12-02T09:34:28.364-06:00","lastCompletedBackup":
91+
"2019-12-02T09:34:28.364-06:00","lastConnected":"2019-12-02T11:02:36.108-06:00","lastMaintenanceDate":
92+
"2021-02-16T07:01:11.697-06:00","lastCompactDate":"2021-02-16T07:01:11.694-06:00","modificationDate":
93+
"2021-02-17T04:57:27.222-06:00","creationDate":"2019-09-26T15:27:38.806-05:00","using":true,
94+
"alertState":16,"alertStates":["CriticalBackupAlert"],"percentComplete":100.0,"storePointId":10989,
95+
"storePointName":"fsa-iad-2","serverId":160024121,"serverGuid":"883282371081742804","serverName":
96+
"fsa-iad","serverHostName":"https://web-fsa-iad.crashplan.com","isProvider":false,"archiveGuid":
97+
"92077743916530001","archiveFormat":"ARCHIVE_V1","activity":{"connected":false,"backingUp":false,
98+
"restoring":false,"timeRemainingInMs":0,"remainingFiles":0,"remainingBytes":0}}]}}"""
8099
TEST_EMPTY_BACKUPUSAGE_RESPONSE = """{"metadata":{"timestamp":"2020-10-13T12:51:28.410Z","params":
81100
{"incBackupUsage":"True","idType":"guid"}},"data":{"computerId":1767,"name":"SNWINTEST1",
82101
"osHostname":"UNKNOWN","guid":"843290890230648046","type":"COMPUTER","status":"Active",
@@ -221,6 +240,75 @@
221240
},
222241
],
223242
}
243+
MATTER_RESPONSE = {
244+
"legalHolds": [
245+
{
246+
"legalHoldUid": "123456789",
247+
"name": "Test legal hold matter",
248+
"description": "",
249+
"notes": None,
250+
"holdExtRef": None,
251+
"active": True,
252+
"creationDate": "2020-08-05T10:49:58.353-05:00",
253+
"lastModified": "2020-08-05T10:49:58.358-05:00",
254+
"creator": {
255+
"userUid": "12345",
256+
"username": "user@code42.com",
257+
"email": "user@code42.com",
258+
"userExtRef": None,
259+
},
260+
"holdPolicyUid": "966191295667423997",
261+
},
262+
{
263+
"legalHoldUid": "987654321",
264+
"name": "Another Matter",
265+
"description": "",
266+
"notes": None,
267+
"holdExtRef": None,
268+
"active": True,
269+
"creationDate": "2020-05-20T15:58:31.375-05:00",
270+
"lastModified": "2020-05-28T13:49:16.098-05:00",
271+
"creator": {
272+
"userUid": "76543",
273+
"username": "user2@code42.com",
274+
"email": "user2@code42.com",
275+
"userExtRef": None,
276+
},
277+
"holdPolicyUid": "946178665645035826",
278+
},
279+
]
280+
}
281+
ALL_CUSTODIANS_RESPONSE = {
282+
"legalHoldMemberships": [
283+
{
284+
"legalHoldMembershipUid": "99999",
285+
"active": True,
286+
"creationDate": "2020-07-16T08:50:23.405Z",
287+
"legalHold": {
288+
"legalHoldUid": "123456789",
289+
"name": "Test legal hold matter",
290+
},
291+
"user": {
292+
"userUid": "840103986007089121",
293+
"username": "ttranda_deactivated@ttrantest.com",
294+
"email": "ttranda_deactivated@ttrantest.com",
295+
"userExtRef": None,
296+
},
297+
},
298+
{
299+
"legalHoldMembershipUid": "88888",
300+
"active": True,
301+
"creationDate": "2020-07-16T08:50:23.405Z",
302+
"legalHold": {"legalHoldUid": "987654321", "name": "Another Matter"},
303+
"user": {
304+
"userUid": "840103986007089121",
305+
"username": "ttranda_deactivated@ttrantest.com",
306+
"email": "ttranda_deactivated@ttrantest.com",
307+
"userExtRef": None,
308+
},
309+
},
310+
]
311+
}
224312

225313

226314
def _create_py42_response(mocker, text):
@@ -274,6 +362,14 @@ def users_list_generator():
274362
yield TEST_USERS_LIST_PAGE
275363

276364

365+
def matter_list_generator():
366+
yield MATTER_RESPONSE
367+
368+
369+
def custodian_list_generator():
370+
yield ALL_CUSTODIANS_RESPONSE
371+
372+
277373
@pytest.fixture
278374
def backupusage_response(mocker):
279375
return _create_py42_response(mocker, TEST_BACKUPUSAGE_RESPONSE)
@@ -356,6 +452,18 @@ def get_all_users_success(cli_state):
356452
cli_state.sdk.users.get_all.return_value = users_list_generator()
357453

358454

455+
@pytest.fixture
456+
def get_all_matter_success(cli_state):
457+
cli_state.sdk.legalhold.get_all_matters.return_value = matter_list_generator()
458+
459+
460+
@pytest.fixture
461+
def get_all_custodian_success(cli_state):
462+
cli_state.sdk.legalhold.get_all_matter_custodians.return_value = (
463+
custodian_list_generator()
464+
)
465+
466+
359467
def test_deactivate_deactivates_device(
360468
runner, cli_state, deactivate_device_success, get_device_by_guid_success
361469
):
@@ -558,6 +666,79 @@ def test_add_usernames_to_device_dataframe_adds_usernames_to_dataframe(
558666
assert "username" in result.columns
559667

560668

669+
def test_add_legal_hold_membership_to_device_dataframe_adds_legal_hold_columns_to_dataframe(
670+
cli_state, get_all_matter_success, get_all_custodian_success
671+
):
672+
testdf = DataFrame.from_records(
673+
[
674+
{"userUid": "840103986007089121", "status": "Active"},
675+
{"userUid": "836473273124890369", "status": "Active, Deauthorized"},
676+
]
677+
)
678+
result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf)
679+
assert "legalHoldUid" in result.columns
680+
assert "legalHoldName" in result.columns
681+
682+
683+
def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivated(
684+
cli_state, get_all_matter_success, get_all_custodian_success
685+
):
686+
testdf = DataFrame.from_records(
687+
[
688+
{"userUid": "840103986007089121", "status": "Deactivated"},
689+
{"userUid": "840103986007089121", "status": "Active"},
690+
]
691+
)
692+
693+
testdf_result = DataFrame.from_records(
694+
[
695+
{
696+
"userUid": "840103986007089121",
697+
"status": "Deactivated",
698+
"legalHoldUid": np.nan,
699+
"legalHoldName": np.nan,
700+
},
701+
{
702+
"userUid": "840103986007089121",
703+
"status": "Active",
704+
"legalHoldUid": "123456789,987654321",
705+
"legalHoldName": "Test legal hold matter,Another Matter",
706+
},
707+
]
708+
)
709+
result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf)
710+
711+
assert_frame_equal(result, testdf_result)
712+
713+
714+
def test_list_include_legal_hold_membership_merges_in_and_concats_legal_hold_info(
715+
runner,
716+
cli_state,
717+
get_all_devices_success,
718+
get_all_custodian_success,
719+
get_all_matter_success,
720+
):
721+
result = runner.invoke(
722+
cli, ["devices", "list", "--include-legal-hold-membership"], obj=cli_state
723+
)
724+
725+
assert "Test legal hold matter,Another Matter" in result.output
726+
assert "123456789,987654321" in result.output
727+
728+
729+
def test_break_backup_usage_into_total_storage_correctly_calculates_values():
730+
test_backupusage_cell = json.loads(TEST_BACKUPUSAGE_RESPONSE)["data"]["backupUsage"]
731+
result = _break_backup_usage_into_total_storage(test_backupusage_cell)
732+
733+
test_empty_backupusage_cell = json.loads(TEST_EMPTY_BACKUPUSAGE_RESPONSE)["data"][
734+
"backupUsage"
735+
]
736+
empty_result = _break_backup_usage_into_total_storage(test_empty_backupusage_cell)
737+
738+
assert_series_equal(result, Series([2, 56968051]))
739+
assert_series_equal(empty_result, Series([0, 0]))
740+
741+
561742
def test_last_connected_after_filters_appropriate_results(
562743
cli_state, runner, get_all_devices_success
563744
):

0 commit comments

Comments
 (0)