From 0cfe41826683b71733002e15ee25739dd2de2a4f Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 17:01:05 -0400 Subject: [PATCH 01/10] Removes TUF-specific formats from formats.py Largely resolves Issue #161, especially given the merge of PR #162. See GitHub: https://github.com/secure-systems-lab/securesystemslib/issues/161 https://github.com/secure-systems-lab/securesystemslib/pull/162 Signed-off-by: Sebastien Awwad --- securesystemslib/formats.py | 240 ++---------------------------------- 1 file changed, 10 insertions(+), 230 deletions(-) diff --git a/securesystemslib/formats.py b/securesystemslib/formats.py index 59637feb..d87de7bb 100755 --- a/securesystemslib/formats.py +++ b/securesystemslib/formats.py @@ -99,15 +99,20 @@ UNIX_TIMESTAMP_SCHEMA = SCHEMA.Integer(lo=0, hi=2147483647) # A hexadecimal value in '23432df87ab..' format. -HASH_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') +HEX_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') + +HASH_SCHEMA = HEX_SCHEMA # A dict in {'sha256': '23432df87ab..', 'sha512': '34324abc34df..', ...} format. HASHDICT_SCHEMA = SCHEMA.DictOf( key_schema = SCHEMA.AnyString(), value_schema = HASH_SCHEMA) -# A hexadecimal value in '23432df87ab..' format. -HEX_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') +# Uniform Resource Locator identifier (e.g., 'https://www.updateframework.com/'). +# TODO: Some level of restriction here would be good.... Note that I pulled +# this from securesystemslib, since it's neither sophisticated nor used +# by anyone else. +URL_SCHEMA = SCHEMA.AnyString() # A key identifier (e.g., a hexadecimal value identifying an RSA key). KEYID_SCHEMA = HASH_SCHEMA @@ -119,31 +124,10 @@ # 'rsassa-pss-sha256' is one of the signing schemes for key type 'rsa'). SCHEME_SCHEMA = SCHEMA.AnyString() -# A relative file path (e.g., 'metadata/root/'). -RELPATH_SCHEMA = SCHEMA.AnyString() -RELPATHS_SCHEMA = SCHEMA.ListOf(RELPATH_SCHEMA) - -# An absolute path. +# A path string, whether relative or absolute, e.g. 'metadata/root/' PATH_SCHEMA = SCHEMA.AnyString() PATHS_SCHEMA = SCHEMA.ListOf(PATH_SCHEMA) -# Uniform Resource Locator identifier (e.g., 'https://www.updateframework.com/'). -URL_SCHEMA = SCHEMA.AnyString() - -# A dictionary holding version information. -VERSION_SCHEMA = SCHEMA.Object( - object_name = 'VERSION_SCHEMA', - major = SCHEMA.Integer(lo=0), - minor = SCHEMA.Integer(lo=0), - fix = SCHEMA.Integer(lo=0)) - -# An integer representing the numbered version of a metadata file. -# Must be 1, or greater. -METADATAVERSION_SCHEMA = SCHEMA.Integer(lo=0) - -# An integer representing length. Must be 0, or greater. -LENGTH_SCHEMA = SCHEMA.Integer(lo=0) - # An integer representing logger levels, such as logging.CRITICAL (=50). # Must be between 0 and 50. LOGLEVEL_SCHEMA = SCHEMA.Integer(lo=0, hi=50) @@ -171,14 +155,6 @@ # A value that is either True or False, on or off, etc. BOOLEAN_SCHEMA = SCHEMA.Boolean() -# A role's threshold value (i.e., the minimum number -# of signatures required to sign a metadata file). -# Must be 1 and greater. -THRESHOLD_SCHEMA = SCHEMA.Integer(lo=1) - -# A string representing a role's name. -ROLENAME_SCHEMA = SCHEMA.AnyString() - # The minimum number of bits for an RSA key. Must be 2048 bits, or greater # (recommended by TUF). Recommended RSA key sizes: # http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 @@ -188,12 +164,6 @@ # default). ECDSA_SCHEME_SCHEMA = SCHEMA.OneOf([SCHEMA.String('ecdsa-sha2-nistp256')]) -# The number of hashed bins, or the number of delegated roles. See -# delegate_hashed_bins() in 'repository_tool.py' for an example. Note: -# Tools may require further restrictions on the number of bins, such -# as requiring them to be a power of 2. -NUMBINS_SCHEMA = SCHEMA.Integer(lo=1) - # A pyca-cryptography signature. PYCACRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyBytes() @@ -316,40 +286,6 @@ keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), keyval = KEYVAL_SCHEMA) -# Information about target files, like file length and file hash(es). This -# schema allows the storage of multiple hashes for the same file (e.g., sha256 -# and sha512 may be computed for the same file and stored). -FILEINFO_SCHEMA = SCHEMA.Object( - object_name = 'FILEINFO_SCHEMA', - length = LENGTH_SCHEMA, - hashes = HASHDICT_SCHEMA, - version = SCHEMA.Optional(METADATAVERSION_SCHEMA), - custom = SCHEMA.Optional(SCHEMA.Object())) - -# Version information specified in "snapshot.json" for each role available on -# the TUF repository. The 'FILEINFO_SCHEMA' object was previously listed in -# the snapshot role, but was switched to this object format to reduce the -# amount of metadata that needs to be downloaded. Listing version numbers in -# "snapshot.json" also prevents rollback attacks for roles that clients have -# not downloaded. -VERSIONINFO_SCHEMA = SCHEMA.Object( - object_name = 'VERSIONINFO_SCHEMA', - version = METADATAVERSION_SCHEMA) - -# A dict holding the version information for a particular metadata role. The -# dict keys hold the relative file paths, and the dict values the corresponding -# version numbers. -VERSIONDICT_SCHEMA = SCHEMA.DictOf( - key_schema = RELPATH_SCHEMA, - value_schema = VERSIONINFO_SCHEMA) - -# A dict holding the information for a particular target / file. The dict keys -# hold the relative file paths, and the dict values the corresponding file -# information. -FILEDICT_SCHEMA = SCHEMA.DictOf( - key_schema = RELPATH_SCHEMA, - value_schema = FILEINFO_SCHEMA) - # A single signature of an object. Indicates the signature, and the KEYID of # the signing key. I debated making the signature schema not contain the key # ID and instead have the signatures of a file be a dictionary with the key @@ -361,22 +297,6 @@ keyid = KEYID_SCHEMA, sig = HEX_SCHEMA) -# List of SIGNATURE_SCHEMA. -SIGNATURES_SCHEMA = SCHEMA.ListOf(SIGNATURE_SCHEMA) - -# A schema holding the result of checking the signatures of a particular -# 'SIGNABLE_SCHEMA' role. -# For example, how many of the signatures for the 'Target' role are -# valid? This SCHEMA holds this information. See 'sig.py' for -# more information. -SIGNATURESTATUS_SCHEMA = SCHEMA.Object( - object_name = 'SIGNATURESTATUS_SCHEMA', - threshold = SCHEMA.Integer(), - good_sigs = KEYIDS_SCHEMA, - bad_sigs = KEYIDS_SCHEMA, - unknown_sigs = KEYIDS_SCHEMA, - untrusted_sigs = KEYIDS_SCHEMA) - # A signable object. Holds the signing role and its associated signatures. SIGNABLE_SCHEMA = SCHEMA.Object( object_name = 'SIGNABLE_SCHEMA', @@ -388,147 +308,7 @@ key_schema = KEYID_SCHEMA, value_schema = KEY_SCHEMA) -# The format used by the key database to store keys. The dict keys hold a key -# identifier and the dict values any object. The key database should store -# key objects in the values (e.g., 'RSAKEY_SCHEMA', 'DSAKEY_SCHEMA'). -KEYDB_SCHEMA = SCHEMA.DictOf( - key_schema = KEYID_SCHEMA, - value_schema = SCHEMA.Any()) - -# A path hash prefix is a hexadecimal string. -PATH_HASH_PREFIX_SCHEMA = HEX_SCHEMA - -# A list of path hash prefixes. -PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) - -# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, -# 'paths':[filepaths..]} format. -ROLE_SCHEMA = SCHEMA.Object( - object_name = 'ROLE_SCHEMA', - name = SCHEMA.Optional(ROLENAME_SCHEMA), - keyids = KEYIDS_SCHEMA, - threshold = THRESHOLD_SCHEMA, - backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA), - paths = SCHEMA.Optional(RELPATHS_SCHEMA), - path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) - -# A dict of roles where the dict keys are role names and the dict values holding -# the role data/information. -ROLEDICT_SCHEMA = SCHEMA.DictOf( - key_schema = ROLENAME_SCHEMA, - value_schema = ROLE_SCHEMA) - -# Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. -ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA) - -# The delegated roles of a Targets role (a parent). -DELEGATIONS_SCHEMA = SCHEMA.Object( - keys = KEYDICT_SCHEMA, - roles = ROLELIST_SCHEMA) - -# The fileinfo format of targets specified in the repository and -# developer tools. The second element of this list holds custom data about the -# target, such as file permissions, author(s), last modified, etc. -CUSTOM_SCHEMA = SCHEMA.Object() - -PATH_FILEINFO_SCHEMA = SCHEMA.DictOf( - key_schema = RELPATH_SCHEMA, - value_schema = CUSTOM_SCHEMA) - -# TUF roledb -ROLEDB_SCHEMA = SCHEMA.Object( - object_name = 'ROLEDB_SCHEMA', - keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), - previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), - version = SCHEMA.Optional(METADATAVERSION_SCHEMA), - expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA), - signatures = SCHEMA.Optional(SIGNATURES_SCHEMA), - paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])), - path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), - delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA), - partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA)) - -# Root role: indicates root keys and top-level roles. -ROOT_SCHEMA = SCHEMA.Object( - object_name = 'ROOT_SCHEMA', - _type = SCHEMA.String('root'), - version = METADATAVERSION_SCHEMA, - consistent_snapshot = BOOLEAN_SCHEMA, - expires = ISO8601_DATETIME_SCHEMA, - keys = KEYDICT_SCHEMA, - roles = ROLEDICT_SCHEMA) - -# Targets role: Indicates targets and delegates target paths to other roles. -TARGETS_SCHEMA = SCHEMA.Object( - object_name = 'TARGETS_SCHEMA', - _type = SCHEMA.String('targets'), - version = METADATAVERSION_SCHEMA, - expires = ISO8601_DATETIME_SCHEMA, - targets = FILEDICT_SCHEMA, - delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA)) - -# Snapshot role: indicates the latest versions of all metadata (except -# timestamp). -SNAPSHOT_SCHEMA = SCHEMA.Object( - object_name = 'SNAPSHOT_SCHEMA', - _type = SCHEMA.String('snapshot'), - version = METADATAVERSION_SCHEMA, - expires = ISO8601_DATETIME_SCHEMA, - meta = VERSIONDICT_SCHEMA) - -# Timestamp role: indicates the latest version of the snapshot file. -TIMESTAMP_SCHEMA = SCHEMA.Object( - object_name = 'TIMESTAMP_SCHEMA', - _type = SCHEMA.String('timestamp'), - version = METADATAVERSION_SCHEMA, - expires = ISO8601_DATETIME_SCHEMA, - meta = FILEDICT_SCHEMA) - -# project.cfg file: stores information about the project in a json dictionary -PROJECT_CFG_SCHEMA = SCHEMA.Object( - object_name = 'PROJECT_CFG_SCHEMA', - project_name = SCHEMA.AnyString(), - layout_type = SCHEMA.OneOf([SCHEMA.String('repo-like'), SCHEMA.String('flat')]), - targets_location = PATH_SCHEMA, - metadata_location = PATH_SCHEMA, - prefix = PATH_SCHEMA, - public_keys = KEYDICT_SCHEMA, - threshold = SCHEMA.Integer(lo = 0, hi = 2) - ) - -# A schema containing information a repository mirror may require, -# such as a url, the path of the directory metadata files, etc. -MIRROR_SCHEMA = SCHEMA.Object( - object_name = 'MIRROR_SCHEMA', - url_prefix = URL_SCHEMA, - metadata_path = RELPATH_SCHEMA, - targets_path = RELPATH_SCHEMA, - confined_target_dirs = RELPATHS_SCHEMA, - custom = SCHEMA.Optional(SCHEMA.Object())) - -# A dictionary of mirrors where the dict keys hold the mirror's name and -# and the dict values the mirror's data (i.e., 'MIRROR_SCHEMA'). -# The repository class of 'updater.py' accepts dictionaries -# of this type provided by the TUF client. -MIRRORDICT_SCHEMA = SCHEMA.DictOf( - key_schema = SCHEMA.AnyString(), - value_schema = MIRROR_SCHEMA) - -# A Mirrorlist: indicates all the live mirrors, and what documents they -# serve. -MIRRORLIST_SCHEMA = SCHEMA.Object( - object_name = 'MIRRORLIST_SCHEMA', - _type = SCHEMA.String('mirrors'), - version = METADATAVERSION_SCHEMA, - expires = ISO8601_DATETIME_SCHEMA, - mirrors = SCHEMA.ListOf(MIRROR_SCHEMA)) - -# Any of the role schemas (e.g., TIMESTAMP_SCHEMA, SNAPSHOT_SCHEMA, etc.) -ANYROLE_SCHEMA = SCHEMA.OneOf([ROOT_SCHEMA, TARGETS_SCHEMA, SNAPSHOT_SCHEMA, - TIMESTAMP_SCHEMA, MIRROR_SCHEMA]) + From dc82cbfe9871802bd5e6538f3f2f832e36c04054 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 15:39:55 -0400 Subject: [PATCH 02/10] Remove unused util.py functions: The following functions in securesystemslib.util are not used anywhere, not in securesystemslib, not in TUF, and not in in-toto: - find_delegated_roles - ensure_all_targets_allowed - paths_are_consistent_with_hash_prefixes In addition, the first two are also clearly TUF-specific (and the third is a close call). So I'm removing them all. This commit also removes their tests. Signed-off-by: Sebastien Awwad --- securesystemslib/util.py | 256 +-------------------------------------- tests/test_util.py | 196 ------------------------------ 2 files changed, 1 insertion(+), 451 deletions(-) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index 8a7dc54b..e65bb71a 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -485,261 +485,7 @@ def file_in_confined_directories(filepath, confined_directories): return False -def find_delegated_role(roles, delegated_role): - """ - - Find the index, if any, of a role with a given name in a list of roles. - - - roles: - The list of roles, each of which must have a 'name' attribute. - - delegated_role: - The name of the role to be found in the list of roles. - - - securesystemslib.exceptions.RepositoryError, if the list of roles has - invalid data. - - - No known side effects. - - - The unique index, an interger, in the list of roles. if 'delegated_role' - does not exist, 'None' is returned. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. Raise - # 'securesystemslib.exceptions.FormatError' if any are improperly formatted. - securesystemslib.formats.ROLELIST_SCHEMA.check_match(roles) - securesystemslib.formats.ROLENAME_SCHEMA.check_match(delegated_role) - - # The index of a role, if any, with the same name. - role_index = None - - for index in six.moves.xrange(len(roles)): - role = roles[index] - name = role.get('name') - - # This role has no name. - if name is None: - no_name_message = 'Role with no name.' - raise securesystemslib.exceptions.RepositoryError(no_name_message) - - # Does this role have the same name? - else: - # This role has the same name, and... - if name == delegated_role: - # ...it is the only known role with the same name. - if role_index is None: - role_index = index - - # ...there are at least two roles with the same name. - else: - duplicate_role_message = 'Duplicate role (' + str(delegated_role) + ').' - raise securesystemslib.exceptions.RepositoryError( - 'Duplicate role (' + str(delegated_role) + ').') - - # This role has a different name. - else: - logger.debug('Skipping delegated role: ' + repr(delegated_role)) - - return role_index - - -def ensure_all_targets_allowed(rolename, list_of_targets, parent_delegations): - """ - - Ensure that the list of targets specified by 'rolename' are allowed; this - is determined by inspecting the 'delegations' field of the parent role of - 'rolename'. If a target specified by 'rolename' is not found in the - delegations field of 'metadata_object_of_parent', raise an exception. The - top-level role 'targets' is allowed to list any target file, so this - function does not raise an exception if 'rolename' is 'targets'. - - Targets allowed are either exlicitly listed under the 'paths' field, or - implicitly exist under a subdirectory of a parent directory listed under - 'paths'. A parent role may delegate trust to all files under a particular - directory, including files in subdirectories, by simply listing the - directory (e.g., '/packages/source/Django/', the equivalent of - '/packages/source/Django/*'). Targets listed in hashed bins are also - validated (i.e., its calculated path hash prefix must be delegated by the - parent role). - - TODO: Should the TUF spec restrict the repository to one particular - algorithm when calcutating path hash prefixes (currently restricted to - SHA256)? Should we allow the repository to specify in the role dictionary - the algorithm used for these generated hashed paths? - - - rolename: - The name of the role whose targets must be verified. This is a - role name and should not end in '.json'. Examples: 'root', 'targets', - 'targets/linux/x86'. - - list_of_targets: - The targets of 'rolename', as listed in targets field of the 'rolename' - metadata. 'list_of_targets' are target paths relative to the targets - directory of the repository. The delegations of the parent role are - checked to verify that the targets of 'list_of_targets' are valid. - - parent_delegations: - The parent delegations of 'rolename'. The metadata object stores - the allowed paths and path hash prefixes of child delegations in its - 'delegations' attribute. - - - securesystemslib.exceptions.FormatError: - If any of the arguments are improperly formatted. - - securesystemslib.exceptions.ForbiddenTargetError: - If the targets of 'metadata_role' are not allowed according to - the parent's metadata file. The 'paths' and 'path_hash_prefixes' - attributes are verified. - - securesystemslib.exceptions.RepositoryError: - If the parent of 'rolename' has not made a delegation to 'rolename'. - - - None. - - - None. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. Raise - # 'securesystemslib.exceptions.FormatError' if any are improperly formatted. - securesystemslib.formats.ROLENAME_SCHEMA.check_match(rolename) - securesystemslib.formats.RELPATHS_SCHEMA.check_match(list_of_targets) - securesystemslib.formats.DELEGATIONS_SCHEMA.check_match(parent_delegations) - - # Return if 'rolename' is 'targets'. 'targets' is not a delegated role. Any - # target file listed in 'targets' is allowed. - if rolename == 'targets': - return - - # The allowed targets of delegated roles are stored in the parent's metadata - # file. Iterate 'list_of_targets' and confirm they are trusted, or their - # root parent directory exists in the role delegated paths, or path hash - # prefixes, of the parent role. First, locate 'rolename' in the 'roles' - # attribute of 'parent_delegations'. - roles = parent_delegations['roles'] - role_index = find_delegated_role(roles, rolename) - - # Ensure the delegated role exists prior to extracting trusted paths from - # the parent's 'paths', or trusted path hash prefixes from the parent's - # 'path_hash_prefixes'. - if role_index is not None: - role = roles[role_index] - allowed_child_paths = role.get('paths') - allowed_child_path_hash_prefixes = role.get('path_hash_prefixes') - actual_child_targets = list_of_targets - - if allowed_child_path_hash_prefixes is not None: - consistent = paths_are_consistent_with_hash_prefixes - - # 'actual_child_tarets' (i.e., 'list_of_targets') should have lenth - # greater than zero due to the format check above. - if not consistent(actual_child_targets, - allowed_child_path_hash_prefixes): - message = repr(rolename) + ' specifies a target that does not' + \ - ' have a path hash prefix listed in its parent role.' - raise securesystemslib.exceptions.ForbiddenTargetError(message) - - elif allowed_child_paths is not None: - # Check that each delegated target is either explicitly listed or a - # parent directory is found under role['paths'], otherwise raise an - # exception. If the parent role explicitly lists target file paths in - # 'paths', this loop will run in O(n^2), the worst-case. The repository - # maintainer will likely delegate entire directories, and opt for - # explicit file paths if the targets in a directory are delegated to - # different roles/developers. - for child_target in actual_child_targets: - for allowed_child_path in allowed_child_paths: - if fnmatch.fnmatch(child_target, allowed_child_path): - break - - else: - raise securesystemslib.exceptions.ForbiddenTargetError( - 'Role ' + repr(rolename) + ' specifies' - ' target' + repr(child_target) + ',' + ' which is not an allowed' - ' path according to the delegations set by its parent role.') - - else: - # 'role' should have been validated when it was downloaded. - # The 'paths' or 'path_hash_prefixes' attributes should not be missing, - # so raise an error in case this clause is reached. - raise securesystemslib.exceptions.FormatError(repr(role) + ' did not' - ' contain one of the required fields ("paths" or' - ' "path_hash_prefixes").') - - # Raise an exception if the parent has not delegated to the specified - # 'rolename' child role. - else: - raise securesystemslib.exceptions.RepositoryError('The parent role has' - ' not delegated to ' + repr(rolename) + '.') - - -def paths_are_consistent_with_hash_prefixes(paths, path_hash_prefixes): - """ - - Determine whether a list of paths are consistent with their alleged path - hash prefixes. By default, the SHA256 hash function is used. - - - paths: - A list of paths for which their hashes will be checked. - - path_hash_prefixes: - The list of path hash prefixes with which to check the list of paths. - - - securesystemslib.exceptions.FormatError: - If the arguments are improperly formatted. - - - No known side effects. - - - A Boolean indicating whether or not the paths are consistent with the - hash prefix. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. Raise - # 'securesystemslib.exceptions.FormatError' if any are improperly formatted. - securesystemslib.formats.RELPATHS_SCHEMA.check_match(paths) - securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA.check_match(path_hash_prefixes) - - # Assume that 'paths' and 'path_hash_prefixes' are inconsistent until - # proven otherwise. - consistent = False - - # The format checks above ensure the 'paths' and 'path_hash_prefix' lists - # have lengths greater than zero. - for path in paths: - path_hash = get_target_hash(path) - - # Assume that every path is inconsistent until proven otherwise. - consistent = False - - for path_hash_prefix in path_hash_prefixes: - if path_hash.startswith(path_hash_prefix): - consistent = True - break - - # This path has no matching path_hash_prefix. Stop looking further. - if not consistent: - break - - return consistent - - +# TODO: Move get_target_hash back to TUF; it's TUF-specific. def get_target_hash(target_filepath): """ diff --git a/tests/test_util.py b/tests/test_util.py index 3d2c2434..b4a7c729 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -403,202 +403,6 @@ def test_C1_get_target_hash(self): - def test_C2_find_delegated_role(self): - # Test normal case. Create an expected role list, which is one of the - # required arguments to 'find_delegated_role()'. - role_list = [ - { - "keyids": [ - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf" - ], - "name": "targets/warehouse", - "paths": [ - "/file1.txt", "/README.txt", '/warehouse/' - ], - "threshold": 3 - }, - { - "keyids": [ - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf" - ], - "name": "targets/tuf", - "paths": [ - "/updater.py", "formats.py", '/tuf/' - ], - "threshold": 4 - } - ] - - self.assertTrue(securesystemslib.formats.ROLELIST_SCHEMA.matches(role_list)) - self.assertEqual(securesystemslib.util.find_delegated_role(role_list, - 'targets/tuf'), 1) - self.assertEqual(securesystemslib.util.find_delegated_role(role_list, - 'targets/warehouse'), 0) - - # Test for non-existent role. 'find_delegated_role()' returns 'None' - # if the role is not found. - self.assertEqual(securesystemslib.util.find_delegated_role(role_list, - 'targets/non-existent'), None) - - # Test improperly formatted arguments. - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.find_delegated_role, 8, role_list) - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.find_delegated_role, 8, 'targets/tuf') - - # Test duplicate roles. - role_list.append(role_list[1]) - self.assertRaises(securesystemslib.exceptions.RepositoryError, - securesystemslib.util.find_delegated_role, role_list, 'targets/tuf') - - # Test missing 'name' attribute (optional, but required by - # 'find_delegated_role()'). - # Delete the duplicate role, and the remaining role's 'name' attribute. - del role_list[2] - del role_list[0]['name'] - self.assertRaises(securesystemslib.exceptions.RepositoryError, - securesystemslib.util.find_delegated_role, role_list, - 'targets/warehouse') - - - - def test_C3_paths_are_consistent_with_hash_prefixes(self): - # Test normal case. - path_hash_prefixes = ['e3a3', '8fae', 'd543'] - list_of_targets = ['/file1.txt', '/README.txt', '/warehouse/file2.txt'] - - # Ensure the paths of 'list_of_targets' each have the expected path hash - # prefix listed in 'path_hash_prefixes'. - for filepath in list_of_targets: - self.assertTrue(securesystemslib.util.get_target_hash( - filepath)[0:4] in path_hash_prefixes) - - self.assertTrue(securesystemslib.util.paths_are_consistent_with_hash_prefixes( - list_of_targets, path_hash_prefixes)) - - extra_invalid_prefix = ['e3a3', '8fae', 'd543', '0000'] - self.assertTrue(securesystemslib.util.paths_are_consistent_with_hash_prefixes( - list_of_targets, extra_invalid_prefix)) - - # Test improperly formatted arguments. - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.paths_are_consistent_with_hash_prefixes, 8, - path_hash_prefixes) - - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.paths_are_consistent_with_hash_prefixes, - list_of_targets, 8) - - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.paths_are_consistent_with_hash_prefixes, - list_of_targets, ['zza1']) - - # Test invalid list of targets. - bad_target_path = '/file5.txt' - self.assertTrue(securesystemslib.util.get_target_hash( - bad_target_path)[0:4] not in path_hash_prefixes) - self.assertFalse(securesystemslib.util.paths_are_consistent_with_hash_prefixes( - [bad_target_path], path_hash_prefixes)) - - # Add invalid target path to 'list_of_targets'. - list_of_targets.append(bad_target_path) - self.assertFalse(securesystemslib.util.paths_are_consistent_with_hash_prefixes( - list_of_targets, path_hash_prefixes)) - - - - def test_C4_ensure_all_targets_allowed(self): - # Test normal case. - rolename = 'targets/warehouse' - self.assertTrue(securesystemslib.formats.ROLENAME_SCHEMA.matches(rolename)) - list_of_targets = ['/file1.txt', '/README.txt', '/warehouse/file2.txt'] - self.assertTrue(securesystemslib.formats.RELPATHS_SCHEMA.matches(list_of_targets)) - parent_delegations = {"keys": { - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf": { - "keytype": "ed25519", - "scheme": "ed25519", - "keyval": { - "public": "3eb81026ded5af2c61fb3d4b272ac53cd1049a810ee88f4df1fc35cdaf918157" - } - } - }, - "roles": [ - { - "keyids": [ - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf" - ], - "name": "targets/warehouse", - "paths": [ - "/file*.txt", "/README.txt", '/warehouse/*' - ], - "threshold": 1 - } - ] - } - self.assertTrue(securesystemslib.formats.DELEGATIONS_SCHEMA.matches(parent_delegations)) - - securesystemslib.util.ensure_all_targets_allowed(rolename, list_of_targets, - parent_delegations) - - # The target files of 'targets' are always allowed. 'list_of_targets' and - # 'parent_delegations' are not checked in this case. - securesystemslib.util.ensure_all_targets_allowed('targets', list_of_targets, - parent_delegations) - - # Test improperly formatted arguments. - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.ensure_all_targets_allowed, 8, list_of_targets, - parent_delegations) - - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.ensure_all_targets_allowed, rolename, 8, - parent_delegations) - - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.ensure_all_targets_allowed, rolename, - list_of_targets, 8) - - # Test for invalid 'rolename', which has not been delegated by its parent, - # 'targets'. - self.assertRaises(securesystemslib.exceptions.RepositoryError, - securesystemslib.util.ensure_all_targets_allowed, - 'targets/non-delegated_rolename', list_of_targets, parent_delegations) - - # Test for target file that is not allowed by the parent role. - self.assertRaises(securesystemslib.exceptions.ForbiddenTargetError, - securesystemslib.util.ensure_all_targets_allowed, 'targets/warehouse', - ['file1.zip'], parent_delegations) - - self.assertRaises(securesystemslib.exceptions.ForbiddenTargetError, - securesystemslib.util.ensure_all_targets_allowed, 'targets/warehouse', - ['file1.txt', 'bad-README.txt'], parent_delegations) - - # Test for required attributes. - # Missing 'paths' attribute. - del parent_delegations['roles'][0]['paths'] - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.ensure_all_targets_allowed, - 'targets/warehouse', list_of_targets, parent_delegations) - - # Test 'path_hash_prefixes' attribute. - path_hash_prefixes = ['e3a3', '8fae', 'd543'] - parent_delegations['roles'][0]['path_hash_prefixes'] = path_hash_prefixes - - # Test normal case for 'path_hash_prefixes'. - securesystemslib.util.ensure_all_targets_allowed('targets/warehouse', - list_of_targets, parent_delegations) - - # Test target file with a path_hash_prefix that is not allowed in its - # parent role. - path_hash_prefix = securesystemslib.util.get_target_hash('file5.txt')[0:4] - self.assertTrue(path_hash_prefix not in parent_delegations['roles'][0] - ['path_hash_prefixes']) - self.assertRaises(securesystemslib.exceptions.ForbiddenTargetError, - securesystemslib.util.ensure_all_targets_allowed, - 'targets/warehouse', ['file5.txt'], parent_delegations) - - - def test_C5_unittest_toolbox_make_temp_directory(self): # Verify that the tearDown function does not fail when # unittest_toolbox.make_temp_directory deletes the generated temp directory From d58c60826e9ac6d8fa75ac9b17f3487758838b67 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 15:41:57 -0400 Subject: [PATCH 03/10] Kill RELPATH_SCHEMA in other modules and correct testing This follows from the commit with subject: "Removes TUF-specific formats from formats.py" It: - removes testing for schemas that were removed by that commit (TUF-specific or otherwise unnecessary schemas) - corrects uses of the deleted RELPATH_SCHEMA to PATH_SCHEMA and RELPATHS_SCHEMA to PATHS_SCHEMA Signed-off-by: Sebastien Awwad --- securesystemslib/hash.py | 2 +- securesystemslib/util.py | 6 +- tests/test_formats.py | 115 +-------------------------------------- tests/test_util.py | 2 +- 4 files changed, 6 insertions(+), 119 deletions(-) diff --git a/securesystemslib/hash.py b/securesystemslib/hash.py index 5892ca27..85eeeda7 100755 --- a/securesystemslib/hash.py +++ b/securesystemslib/hash.py @@ -358,7 +358,7 @@ def digest_filename(filename, algorithm=DEFAULT_HASH_ALGORITHM, """ # Are the arguments properly formatted? If not, raise # 'securesystemslib.exceptions.FormatError'. - securesystemslib.formats.RELPATH_SCHEMA.check_match(filename) + securesystemslib.formats.PATH_SCHEMA.check_match(filename) securesystemslib.formats.NAME_SCHEMA.check_match(algorithm) securesystemslib.formats.NAME_SCHEMA.check_match(hash_library) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index e65bb71a..3f1b4071 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -460,8 +460,8 @@ def file_in_confined_directories(filepath, confined_directories): # Do the arguments have the correct format? # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.RELPATH_SCHEMA.check_match(filepath) - securesystemslib.formats.RELPATHS_SCHEMA.check_match(confined_directories) + securesystemslib.formats.PATH_SCHEMA.check_match(filepath) + securesystemslib.formats.PATHS_SCHEMA.check_match(confined_directories) for confined_directory in confined_directories: # The empty string (arbitrarily chosen) signifies the client is confined @@ -516,7 +516,7 @@ def get_target_hash(target_filepath): # Ensure the arguments have the appropriate number of objects and object # types, and that all dict keys are properly named. # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.RELPATH_SCHEMA.check_match(target_filepath) + securesystemslib.formats.PATH_SCHEMA.check_match(target_filepath) # Calculate the hash of the filepath to determine which bin to find the # target. The client currently assumes the repository uses diff --git a/tests/test_formats.py b/tests/test_formats.py index 31f8ea8a..984cd453 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -69,12 +69,6 @@ def test_schemas(self): 'SCHEME_SCHEMA': (securesystemslib.formats.SCHEME_SCHEMA, 'ecdsa-sha2-nistp256'), - 'RELPATH_SCHEMA': (securesystemslib.formats.RELPATH_SCHEMA, - 'metadata/root/'), - - 'RELPATHS_SCHEMA': (securesystemslib.formats.RELPATHS_SCHEMA, - ['targets/role1/', 'targets/role2/']), - 'PATH_SCHEMA': (securesystemslib.formats.PATH_SCHEMA, '/home/someuser/'), @@ -84,21 +78,12 @@ def test_schemas(self): 'URL_SCHEMA': (securesystemslib.formats.URL_SCHEMA, 'https://www.updateframework.com/'), - 'VERSION_SCHEMA': (securesystemslib.formats.VERSION_SCHEMA, - {'major': 1, 'minor': 0, 'fix': 8}), - - 'LENGTH_SCHEMA': (securesystemslib.formats.LENGTH_SCHEMA, 8), - 'NAME_SCHEMA': (securesystemslib.formats.NAME_SCHEMA, 'Marty McFly'), 'TEXT_SCHEMA': (securesystemslib.formats.TEXT_SCHEMA, 'Password: '), 'BOOLEAN_SCHEMA': (securesystemslib.formats.BOOLEAN_SCHEMA, True), - 'THRESHOLD_SCHEMA': (securesystemslib.formats.THRESHOLD_SCHEMA, 1), - - 'ROLENAME_SCHEMA': (securesystemslib.formats.ROLENAME_SCHEMA, 'Root'), - 'RSAKEYBITS_SCHEMA': (securesystemslib.formats.RSAKEYBITS_SCHEMA, 4096), 'PASSWORD_SCHEMA': (securesystemslib.formats.PASSWORD_SCHEMA, 'secret'), @@ -175,29 +160,11 @@ def test_schemas(self): 'keyval': {'public': 'pubkey', 'private': 'privkey'}}), - 'FILEINFO_SCHEMA': (securesystemslib.formats.FILEINFO_SCHEMA, - {'length': 1024, - 'hashes': {'sha256': 'A4582BCF323BCEF'}, - 'custom': {'type': 'paintjob'}}), - - 'FILEDICT_SCHEMA': (securesystemslib.formats.FILEDICT_SCHEMA, - {'metadata/root.json': {'length': 1024, - 'hashes': {'sha256': 'ABCD123'}, - 'custom': {'type': 'metadata'}}}), - 'SIGNATURE_SCHEMA': (securesystemslib.formats.SIGNATURE_SCHEMA, {'keyid': '123abc', 'method': 'evp', 'sig': 'A4582BCF323BCEF'}), - 'SIGNATURESTATUS_SCHEMA': (securesystemslib.formats.SIGNATURESTATUS_SCHEMA, - {'threshold': 1, - 'good_sigs': ['123abc'], - 'bad_sigs': ['123abc'], - 'unknown_sigs': ['123abc'], - 'untrusted_sigs': ['123abc'], - 'unknown_method_sigs': ['123abc']}), - 'SIGNABLE_SCHEMA': (securesystemslib.formats.SIGNABLE_SCHEMA, {'signed': 'signer', 'signatures': [{'keyid': '123abc', @@ -207,87 +174,7 @@ def test_schemas(self): 'KEYDICT_SCHEMA': (securesystemslib.formats.KEYDICT_SCHEMA, {'123abc': {'keytype': 'rsa', 'scheme': 'rsassa-pss-sha256', - 'keyval': {'public': 'pubkey', 'private': 'privkey'}}}), - - 'KEYDB_SCHEMA': (securesystemslib.formats.KEYDB_SCHEMA, - {'123abc': {'keytype': 'rsa', - 'keyid': '123456789abcdef', - 'keyval': {'public': 'pubkey', 'private': 'privkey'}}}), - - 'ROLE_SCHEMA': (securesystemslib.formats.ROLE_SCHEMA, - {'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}), - - 'ROLEDICT_SCHEMA': (securesystemslib.formats.ROLEDICT_SCHEMA, - {'root': {'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}}), - - 'ROOT_SCHEMA': (securesystemslib.formats.ROOT_SCHEMA, - {'_type': 'root', - 'version': 8, - 'consistent_snapshot': False, - 'expires': '1985-10-21T13:20:00Z', - 'keys': {'123abc': {'keytype': 'rsa', - 'scheme': 'rsassa-pss-sha256', - 'keyval': {'public': 'pubkey', - 'private': 'privkey'}}}, - 'roles': {'root': {'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}}}), - - 'TARGETS_SCHEMA': (securesystemslib.formats.TARGETS_SCHEMA, - {'_type': 'targets', - 'version': 8, - 'expires': '1985-10-21T13:20:00Z', - 'targets': {'metadata/targets.json': {'length': 1024, - 'hashes': {'sha256': 'ABCD123'}, - 'custom': {'type': 'metadata'}}}, - 'delegations': {'keys': {'123abc': {'keytype':'rsa', - 'scheme': 'rsassa-pss-sha256', - 'keyval': {'public': 'pubkey', - 'private': 'privkey'}}}, - 'roles': [{'name': 'root', 'keyids': ['123abc'], - 'threshold': 1, - 'paths': ['path1/', 'path2']}]}}), - - 'SNAPSHOT_SCHEMA': (securesystemslib.formats.SNAPSHOT_SCHEMA, - {'_type': 'snapshot', - 'version': 8, - 'expires': '1985-10-21T13:20:00Z', - 'meta': {'snapshot.json': {'version': 1024}}}), - - 'TIMESTAMP_SCHEMA': (securesystemslib.formats.TIMESTAMP_SCHEMA, - {'_type': 'timestamp', - 'version': 8, - 'expires': '1985-10-21T13:20:00Z', - 'meta': {'metadattimestamp.json': {'length': 1024, - 'hashes': {'sha256': 'AB1245'}}}}), - - 'MIRROR_SCHEMA': (securesystemslib.formats.MIRROR_SCHEMA, - {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata/', - 'targets_path': 'targets/', - 'confined_target_dirs': ['path1/', 'path2/'], - 'custom': {'type': 'mirror'}}), - - 'MIRRORDICT_SCHEMA': (securesystemslib.formats.MIRRORDICT_SCHEMA, - {'mirror1': {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata/', - 'targets_path': 'targets/', - 'confined_target_dirs': ['path1/', 'path2/'], - 'custom': {'type': 'mirror'}}}), - - 'MIRRORLIST_SCHEMA': (securesystemslib.formats.MIRRORLIST_SCHEMA, - {'_type': 'mirrors', - 'version': 8, - 'expires': '1985-10-21T13:20:00Z', - 'mirrors': [{'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata/', - 'targets_path': 'targets/', - 'confined_target_dirs': ['path1/', 'path2/'], - 'custom': {'type': 'mirror'}}]})} + 'keyval': {'public': 'pubkey', 'private': 'privkey'}}})} # Iterate 'valid_schemas', ensuring each 'valid_schema' correctly matches # its respective 'schema_type'. diff --git a/tests/test_util.py b/tests/test_util.py index b4a7c729..d13bbb0e 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -393,7 +393,7 @@ def test_C1_get_target_hash(self): '/warehouse/file2.txt': 'd543a573a2cec67026eff06e75702303559e64e705eba06f65799baaf0424417' } for filepath, target_hash in six.iteritems(expected_target_hashes): - self.assertTrue(securesystemslib.formats.RELPATH_SCHEMA.matches(filepath)) + self.assertTrue(securesystemslib.formats.PATH_SCHEMA.matches(filepath)) self.assertTrue(securesystemslib.formats.HASH_SCHEMA.matches(target_hash)) self.assertEqual(securesystemslib.util.get_target_hash(filepath), target_hash) From 3c7fc3467d8b6c13b8e51c3ceb16b9b77a6af0e8 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 15:52:41 -0400 Subject: [PATCH 04/10] Remove no-longer-needed imports in util.py sys and fnmatch are no longer needed due to the removal of unused functions in prior commits. Signed-off-by: Sebastien Awwad --- securesystemslib/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index 3f1b4071..d09013b8 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -27,12 +27,10 @@ from __future__ import unicode_literals import os -import sys import gzip import shutil import logging import tempfile -import fnmatch import securesystemslib.exceptions import securesystemslib.settings From 913f085119e40b6c9a4d19a67ed7949296f432d0 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 15:53:48 -0400 Subject: [PATCH 05/10] minor: add TODOs for possible futher removals from util.py Signed-off-by: Sebastien Awwad --- securesystemslib/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index d09013b8..da7cbd32 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -103,6 +103,7 @@ def __init__(self, prefix='tuf_temp_'): self._default_temporary_directory(prefix) + # TODO: No code across ssl, tuf, and in-toto uses this function. Remove? def get_compressed_length(self): """ @@ -251,6 +252,7 @@ def seek(self, *args): self.temporary_file.seek(*args) + # TODO: No code across ssl, tuf, and in-toto uses this function. Remove? def decompress_temp_file_object(self, compression): """ From d5e0981a4fa34e08fea487a0e880b6925d48f4f6 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 16:44:35 -0400 Subject: [PATCH 06/10] Removes a host of unused or TUF-specific exceptions and also marks the DecompressionError exception for possible future removal (alongside the compression-related functions). Signed-off-by: Sebastien Awwad --- securesystemslib/exceptions.py | 57 ++-------------------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/securesystemslib/exceptions.py b/securesystemslib/exceptions.py index 7c8e5745..1255ef17 100755 --- a/securesystemslib/exceptions.py +++ b/securesystemslib/exceptions.py @@ -75,47 +75,18 @@ def __str__(self): ') != expected hash (' + repr(self.expected_hash)+')' -class BadVersionNumberError(Error): - """Indicate an error for metadata that contains an invalid version number.""" - - class BadPasswordError(Error): """Indicate an error after encountering an invalid password.""" pass -class UnknownKeyError(Error): - """Indicate an error while verifying key-like objects (e.g., keyids).""" - pass - - -class RepositoryError(Error): - """Indicate an error with a repository's state, such as a missing file.""" - pass - - -class InsufficientKeysError(Error): - """Indicate that metadata role lacks a threshold of pubic or private keys.""" - pass - - -class ForbiddenTargetError(RepositoryError): - """Indicate that a role signed for a target that it was not delegated to.""" - pass - - -class ExpiredMetadataError(Error): - """Indicate that a Metadata file has expired.""" - pass - - class CryptoError(Error): """Indicate any cryptography-related errors.""" pass class BadSignatureError(CryptoError): - """Indicate that some metadata file has a bad signature.""" + """Indicate that some metadata has a bad signature.""" def __init__(self, metadata_role_name): self.metadata_role_name = metadata_role_name @@ -134,6 +105,7 @@ class UnsupportedLibraryError(Error): pass +# TODO: Consider removal alongside the compression functions. class DecompressionError(Error): """Indicate that some error happened while decompressing a file.""" @@ -146,31 +118,6 @@ def __str__(self): return repr(self.exception) -class DownloadError(Error): - """Indicate an error occurred while attempting to download a file.""" - pass - - -class KeyAlreadyExistsError(Error): - """Indicate that a key already exists and cannot be added.""" - pass - - -class RoleAlreadyExistsError(Error): - """Indicate that a role already exists and cannot be added.""" - pass - - -class UnknownRoleError(Error): - """Indicate an error trying to locate or identify a specified role.""" - pass - - -class UnknownTargetError(Error): - """Indicate an error trying to locate or identify a specified target.""" - pass - - class InvalidNameError(Error): """Indicate an error while trying to validate any type of named object.""" pass From 560f66ceab28b5d315667be996cb3b21017df35e Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 4 Sep 2019 18:14:03 +0200 Subject: [PATCH 07/10] Remove/re-word TUF code comments --- securesystemslib/ed25519_keys.py | 2 +- securesystemslib/formats.py | 47 +++++++++++++------------- securesystemslib/hash.py | 12 +++---- securesystemslib/pyca_crypto_keys.py | 50 ++++++++++++++-------------- securesystemslib/schema.py | 4 +-- securesystemslib/util.py | 9 +++-- setup.py | 8 ++--- 7 files changed, 66 insertions(+), 66 deletions(-) diff --git a/securesystemslib/ed25519_keys.py b/securesystemslib/ed25519_keys.py index caf3233d..3eb4860c 100755 --- a/securesystemslib/ed25519_keys.py +++ b/securesystemslib/ed25519_keys.py @@ -100,7 +100,7 @@ except (ImportError, IOError): # pragma: no cover pass -# The optimized pure Python implementation of Ed25519 provided by TUF. If +# The optimized pure Python implementation of Ed25519. If # PyNaCl cannot be imported and an attempt to use is made in this module, a # 'securesystemslib.exceptions.UnsupportedLibraryError' exception is raised. import securesystemslib._vendor.ed25519.ed25519 diff --git a/securesystemslib/formats.py b/securesystemslib/formats.py index d87de7bb..b08e9853 100755 --- a/securesystemslib/formats.py +++ b/securesystemslib/formats.py @@ -15,14 +15,14 @@ See LICENSE for licensing information. - A central location for all format-related checking of TUF objects. - Note: 'formats.py' depends heavily on 'schema.py', so the 'schema.py' - module should be read and understood before tackling this module. + A central location for all format-related checking of securesystemslib + objects. Note: 'formats.py' depends heavily on 'schema.py', so the + 'schema.py' module should be read and understood before tackling this module. 'formats.py' can be broken down into three sections. (1) Schemas and object matching. (2) Classes that represent Role Metadata and help produce - correctly formatted files. (3) Functions that help produce or verify TUF - objects. + correctly formatted files. (3) Functions that help produce or verify + securesystemslib objects. The first section deals with schemas and object matching based on format. There are two ways of checking the format of objects. The first method @@ -57,7 +57,7 @@ schema to ensure correctly formatted metadata. The last section contains miscellaneous functions related to the format of - TUF objects. + securesystemslib objects. Example: signable_object = make_signable(unsigned_object) @@ -148,7 +148,7 @@ SCHEMA.String('sha224'), SCHEMA.String('sha256'), SCHEMA.String('sha384'), SCHEMA.String('sha512')])) -# The contents of an encrypted TUF key. Encrypted TUF keys are saved to files +# The contents of an encrypted key. Encrypted keys are saved to files # in this format. ENCRYPTEDKEY_SCHEMA = SCHEMA.AnyString() @@ -194,13 +194,13 @@ public = SCHEMA.AnyString(), private = SCHEMA.Optional(SCHEMA.String(""))) -# Supported TUF key types. +# Supported securesystemslib key types. KEYTYPE_SCHEMA = SCHEMA.OneOf( [SCHEMA.String('rsa'), SCHEMA.String('ed25519'), SCHEMA.String('ecdsa-sha2-nistp256')]) -# A generic TUF key. All TUF keys should be saved to metadata files in this -# format. +# A generic securesystemslib key. All securesystemslib keys should be saved to +# metadata files in this format. KEY_SCHEMA = SCHEMA.Object( object_name = 'KEY_SCHEMA', keytype = SCHEMA.AnyString(), @@ -218,8 +218,9 @@ keyval = PUBLIC_KEYVAL_SCHEMA, expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA)) -# A TUF key object. This schema simplifies validation of keys that may be one -# of the supported key types. Supported key types: 'rsa', 'ed25519'. +# A securesystemslib key object. This schema simplifies validation of keys +# that may be one of the supported key types. Supported key types: 'rsa', +# 'ed25519'. ANYKEY_SCHEMA = SCHEMA.Object( object_name = 'ANYKEY_SCHEMA', keytype = KEYTYPE_SCHEMA, @@ -229,7 +230,7 @@ keyval = KEYVAL_SCHEMA, expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA)) -# A list of TUF key objects. +# A list of securesystemslib key objects. ANYKEYLIST_SCHEMA = SCHEMA.ListOf(ANYKEY_SCHEMA) # RSA signature schemes. @@ -237,7 +238,7 @@ SCHEMA.RegularExpression(r'rsassa-pss-(md5|sha1|sha224|sha256|sha384|sha512)'), SCHEMA.RegularExpression(r'rsa-pkcs1v15-(md5|sha1|sha224|sha256|sha384|sha512)')]) -# An RSA TUF key. +# An RSA securesystemslib key. RSAKEY_SCHEMA = SCHEMA.Object( object_name = 'RSAKEY_SCHEMA', keytype = SCHEMA.String('rsa'), @@ -246,7 +247,7 @@ keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), keyval = KEYVAL_SCHEMA) -# An ECDSA TUF key. +# An ECDSA securesystemslib key. ECDSAKEY_SCHEMA = SCHEMA.Object( object_name = 'ECDSAKEY_SCHEMA', keytype = SCHEMA.String('ecdsa-sha2-nistp256'), @@ -277,7 +278,7 @@ # supported. ED25519_SIG_SCHEMA = SCHEMA.OneOf([SCHEMA.String('ed25519')]) -# An ed25519 TUF key. +# An ed25519 key. ED25519KEY_SCHEMA = SCHEMA.Object( object_name = 'ED25519KEY_SCHEMA', keytype = SCHEMA.String('ed25519'), @@ -543,13 +544,13 @@ def encode_canonical(object, output_function=None): or joined into a string and returned. Note: This function should be called prior to computing the hash or - signature of a JSON object in TUF. For example, generating a signature - of a signing role object such as 'ROOT_SCHEMA' is required to ensure - repeatable hashes are generated across different json module versions - and platforms. Code elsewhere is free to dump JSON objects in any format - they wish (e.g., utilizing indentation and single quotes around object - keys). These objects are only required to be in "canonical JSON" format - when their hashes or signatures are needed. + signature of a JSON object in securesystemslib. For example, generating a + signature of a signing role object such as 'ROOT_SCHEMA' is required to + ensure repeatable hashes are generated across different json module + versions and platforms. Code elsewhere is free to dump JSON objects in any + format they wish (e.g., utilizing indentation and single quotes around + object keys). These objects are only required to be in "canonical JSON" + format when their hashes or signatures are needed. >>> encode_canonical("") '""' diff --git a/securesystemslib/hash.py b/securesystemslib/hash.py index 85eeeda7..6811d44c 100755 --- a/securesystemslib/hash.py +++ b/securesystemslib/hash.py @@ -14,12 +14,12 @@ Support secure hashing and message digests. Any hash-related routines that - TUF requires should be located in this module. Simplifying the creation of - digest objects, and providing a central location for hash routines are the - main goals of this module. Support routines implemented include functions to - create digest objects given a filename or file object. Only the standard - hashlib library is currently supported, but pyca/cryptography support will be - added in the future. + securesystemslib requires should be located in this module. Simplifying the + creation of digest objects, and providing a central location for hash + routines are the main goals of this module. Support routines implemented + include functions to create digest objects given a filename or file object. + Only the standard hashlib library is currently supported, but + pyca/cryptography support will be added in the future. """ # Help with Python 3 compatibility, where the print statement is a function, an diff --git a/securesystemslib/pyca_crypto_keys.py b/securesystemslib/pyca_crypto_keys.py index 29b87ac0..8c4cf2d5 100755 --- a/securesystemslib/pyca_crypto_keys.py +++ b/securesystemslib/pyca_crypto_keys.py @@ -40,8 +40,8 @@ https://en.wikipedia.org/wiki/PBKDF http://en.wikipedia.org/wiki/Scrypt - TUF key files are encrypted with the AES-256-CTR-Mode symmetric key - algorithm. User passwords are strengthened with PBKDF2, currently set to + securesystemslib key files are encrypted with the AES-256-CTR-Mode symmetric + key algorithm. User passwords are strengthened with PBKDF2, currently set to 100,000 passphrase iterations. The previous evpy implementation used 1,000 iterations. @@ -98,7 +98,7 @@ # Import pyca/cryptography's Key Derivation Function (KDF) module. # 'securesystemslib.keys.py' needs this module to derive a secret key according # to the Password-Based Key Derivation Function 2 specification. The derived -# key is used as the symmetric key to encrypt TUF key information. +# key is used as the symmetric key to encrypt securesystemslib key information. # PKCS#5 v2.0 PBKDF2 specification: http://tools.ietf.org/html/rfc2898#section-5.2 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC @@ -573,7 +573,7 @@ def create_rsa_public_and_private_from_pem(pem, passphrase=None): strengthened'passphrase', and 3DES with CBC mode for encryption/decryption. Alternatively, key data may be encrypted with AES-CTR-Mode and the passphrase strengthened with PBKDF2+SHA256, although this method is used - only with TUF encrypted key files. + only with securesystemslib encrypted key files. >>> public, private = generate_rsa_public_and_private(2048) >>> passphrase = 'secret' @@ -679,16 +679,16 @@ def encrypt_key(key_object, password): Return a string containing 'key_object' in encrypted form. Encrypted strings may be safely saved to a file. The corresponding decrypt_key() function can be applied to the encrypted string to restore the original key - object. 'key_object' is a TUF key (e.g., RSAKEY_SCHEMA, + object. 'key_object' is a securesystemslib key (e.g., RSAKEY_SCHEMA, ED25519KEY_SCHEMA). This function calls the pyca/cryptography library to perform the encryption and derive a suitable encryption key. Whereas an encrypted PEM file uses the Triple Data Encryption Algorithm (3DES), the Cipher-block chaining (CBC) mode of operation, and the Password Based Key Derivation Function 1 (PBKF1) + MD5 to strengthen 'password', - encrypted TUF keys use AES-256-CTR-Mode and passwords strengthened with - PBKDF2-HMAC-SHA256 (100K iterations by default, but may be overriden in - 'settings.PBKDF2_ITERATIONS' by the user). + encrypted securesystemslib keys use AES-256-CTR-Mode and passwords + strengthened with PBKDF2-HMAC-SHA256 (100K iterations by default, but may + be overriden in 'settings.PBKDF2_ITERATIONS' by the user). http://en.wikipedia.org/wiki/Advanced_Encryption_Standard http://en.wikipedia.org/wiki/CTR_mode#Counter_.28CTR.29 @@ -709,8 +709,8 @@ def encrypt_key(key_object, password): key_object: - The TUF key object that should contain the private portion of the ED25519 - key. + The securesystemslib key object that should contain the private portion + of the ED25519 key. password: The password, or passphrase, to encrypt the private part of the RSA @@ -722,8 +722,8 @@ def encrypt_key(key_object, password): improperly formatted or 'key_object' does not contain the private portion of the key. - securesystemslib.exceptions.CryptoError, if an Ed25519 key in encrypted TUF - format cannot be created. + securesystemslib.exceptions.CryptoError, if an Ed25519 key in encrypted + securesystemslib format cannot be created. pyca/Cryptography cryptographic operations called to perform the actual @@ -774,13 +774,13 @@ def decrypt_key(encrypted_key, password): Return a string containing 'encrypted_key' in non-encrypted form. The decrypt_key() function can be applied to the encrypted string to restore - the original key object, a TUF key (e.g., RSAKEY_SCHEMA, ED25519KEY_SCHEMA). - This function calls the appropriate cryptography module (i.e., - pyca_crypto_keys.py) to perform the decryption. + the original key object, a securesystemslib key (e.g., RSAKEY_SCHEMA, + ED25519KEY_SCHEMA). This function calls the appropriate cryptography module + (i.e., pyca_crypto_keys.py) to perform the decryption. - Encrypted TUF keys use AES-256-CTR-Mode and passwords strengthened with - PBKDF2-HMAC-SHA256 (100K iterations be default, but may be overriden in - 'settings.py' by the user). + Encrypted securesystemslib keys use AES-256-CTR-Mode and passwords + strengthened with PBKDF2-HMAC-SHA256 (100K iterations be default, but may + be overriden in 'settings.py' by the user). http://en.wikipedia.org/wiki/Advanced_Encryption_Standard http://en.wikipedia.org/wiki/CTR_mode#Counter_.28CTR.29 @@ -804,9 +804,9 @@ def decrypt_key(encrypted_key, password): encrypted_key: - An encrypted TUF key (additional data is also included, such as salt, - number of password iterations used for the derived encryption key, etc) - of the form 'securesystemslib.formats.ENCRYPTEDKEY_SCHEMA'. + An encrypted securesystemslib key (additional data is also included, such + as salt, number of password iterations used for the derived encryption + key, etc) of the form 'securesystemslib.formats.ENCRYPTEDKEY_SCHEMA'. 'encrypted_key' should have been generated with encrypted_key(). password: @@ -818,11 +818,11 @@ def decrypt_key(encrypted_key, password): securesystemslib.exceptions.FormatError, if the arguments are improperly formatted. - securesystemslib.exceptions.CryptoError, if a TUF key cannot be decrypted - from 'encrypted_key'. + securesystemslib.exceptions.CryptoError, if a securesystemslib key cannot + be decrypted from 'encrypted_key'. - securesystemslib.exceptions.Error, if a valid TUF key object is not found in - 'encrypted_key'. + securesystemslib.exceptions.Error, if a valid securesystemslib key object + is not found in 'encrypted_key'. The pyca/cryptography is library called to perform the actual decryption diff --git a/securesystemslib/schema.py b/securesystemslib/schema.py index f380e92e..8049c5c4 100755 --- a/securesystemslib/schema.py +++ b/securesystemslib/schema.py @@ -38,8 +38,8 @@ 'schema.py' provides additional schemas for testing objects based on other criteria. See 'securesystemslib.formats.py' and the rest of this module for - extensive examples. Anything related to the checking of TUF objects and - their formats can be found in 'formats.py'. + extensive examples. Anything related to the checking of securesystemslib + objects and their formats can be found in 'formats.py'. """ # Help with Python 3 compatibility, where the print statement is a function, an diff --git a/securesystemslib/util.py b/securesystemslib/util.py index da7cbd32..33d3d126 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -44,7 +44,6 @@ # to the filenames of consistent snapshots. HASH_FUNCTION = 'sha256' -# See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('securesystemslib_util') @@ -470,12 +469,12 @@ def file_in_confined_directories(filepath, confined_directories): return True # Normalized paths needed, to account for up-level references, etc. - # TUF clients have the option of setting the list of directories in + # callers have the option of setting the list of directories in # 'confined_directories'. filepath = os.path.normpath(filepath) confined_directory = os.path.normpath(confined_directory) - # A TUF client may restrict himself to specific directories on the + # A caller may restrict himself to specific directories on the # remote repository. The list of paths in 'confined_path', not including # each path's subdirectories, are the only directories the client will # download targets from. @@ -557,11 +556,11 @@ def import_json(): return _json_module else: + # TODO: Drop Python < 2.6 case handling try: module = __import__('json') - # The 'json' module is available in Python > 2.6, and thus this exception - # should not occur in all supported Python installations (> 2.6) of TUF. + # should not occur in all supported Python installations (> 2.6). except ImportError: #pragma: no cover raise ImportError('Could not import the json module') diff --git a/setup.py b/setup.py index a8b5a4a1..2e7cfd81 100755 --- a/setup.py +++ b/setup.py @@ -16,9 +16,9 @@ BUILD SOURCE DISTRIBUTION - The following shell command generates a TUF source archive that can be - distributed to other users. The packaged source is saved to the 'dist' - folder in the current directory. + The following shell command generates a securesystemslib source archive that + can be distributed to other users. The packaged source is saved to the + 'dist' folder in the current directory. $ python setup.py sdist @@ -54,7 +54,7 @@ Note: The last two installation options may require modification of Python's search path (i.e., 'sys.path') or updating an OS environment variable. For example, installing to the user site-packages directory might - result in the installation of TUF scripts to '~/.local/bin'. The user may + result in the installation of scripts to '~/.local/bin'. The user may then be required to update his $PATH variable: $ export PATH=$PATH:~/.local/bin """ From 318f90aa6c0a718ea4b8555d0e22696e0b20f507 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 5 Sep 2019 11:02:52 +0200 Subject: [PATCH 08/10] Add comment to remove TUF-specific TempFile prefix --- securesystemslib/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index 33d3d126..4f67dd76 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -68,6 +68,8 @@ def _default_temporary_directory(self, prefix): raise securesystemslib.exceptions.Error(err) + # TODO: Is it safe to de-TUF the prefix? TUF heavily uses `TempFile` without + # ever overriding `prefix`, thus people might expect the default prefix. def __init__(self, prefix='tuf_temp_'): """ From 532fc2046cbc3704cd4874080079bad139249b3d Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 5 Sep 2019 11:49:15 +0200 Subject: [PATCH 09/10] Remove TUF-specific util.get_target_hash function Also remove now obsolete util.HASH_FUNCTION and corresponding tests. The function is added back to tuf in theupdateframework/tuf#909. --- securesystemslib/util.py | 46 ---------------------------------------- tests/test_util.py | 18 ---------------- 2 files changed, 64 deletions(-) diff --git a/securesystemslib/util.py b/securesystemslib/util.py index 4f67dd76..fd7ce0c4 100755 --- a/securesystemslib/util.py +++ b/securesystemslib/util.py @@ -39,11 +39,6 @@ import six -# The algorithm used by the repository to generate the digests of the -# target filepaths, which are included in metadata files and may be prepended -# to the filenames of consistent snapshots. -HASH_FUNCTION = 'sha256' - logger = logging.getLogger('securesystemslib_util') @@ -486,48 +481,7 @@ def file_in_confined_directories(filepath, confined_directories): return False -# TODO: Move get_target_hash back to TUF; it's TUF-specific. -def get_target_hash(target_filepath): - """ - - Compute the hash of 'target_filepath'. This is useful in conjunction with - the "path_hash_prefixes" attribute in a delegated targets role, which tells - us which paths it is implicitly responsible for. - - The repository may optionally organize targets into hashed bins to ease - target delegations and role metadata management. The use of consistent - hashing allows for a uniform distribution of targets into bins. - - - target_filepath: - The path to the target file on the repository. This will be relative to - the 'targets' (or equivalent) directory on a given mirror. - - - None. - - - None. - - - The hash of 'target_filepath'. - """ - - # Does 'target_filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.PATH_SCHEMA.check_match(target_filepath) - - # Calculate the hash of the filepath to determine which bin to find the - # target. The client currently assumes the repository uses - # 'HASH_FUNCTION' to generate hashes and 'utf-8'. - digest_object = securesystemslib.hash.digest(HASH_FUNCTION) - encoded_target_filepath = target_filepath.encode('utf-8') - digest_object.update(encoded_target_filepath) - target_filepath_hash = digest_object.hexdigest() - return target_filepath_hash _json_module = None diff --git a/tests/test_util.py b/tests/test_util.py index d13bbb0e..c3bdd555 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -385,24 +385,6 @@ def test_B6_load_json_file(self): - def test_C1_get_target_hash(self): - # Test normal case. - expected_target_hashes = { - '/file1.txt': 'e3a3d89eb3b70ce3fbce6017d7b8c12d4abd5635427a0e8a238f53157df85b3d', - '/README.txt': '8faee106f1bb69f34aaf1df1e3c2e87d763c4d878cb96b91db13495e32ceb0b0', - '/warehouse/file2.txt': 'd543a573a2cec67026eff06e75702303559e64e705eba06f65799baaf0424417' - } - for filepath, target_hash in six.iteritems(expected_target_hashes): - self.assertTrue(securesystemslib.formats.PATH_SCHEMA.matches(filepath)) - self.assertTrue(securesystemslib.formats.HASH_SCHEMA.matches(target_hash)) - self.assertEqual(securesystemslib.util.get_target_hash(filepath), target_hash) - - # Test for improperly formatted argument. - self.assertRaises(securesystemslib.exceptions.FormatError, - securesystemslib.util.get_target_hash, 8) - - - def test_C5_unittest_toolbox_make_temp_directory(self): # Verify that the tearDown function does not fail when # unittest_toolbox.make_temp_directory deletes the generated temp directory From 4adbc5a92f138147464d96136a7365c165e88d43 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 5 Sep 2019 13:03:34 +0200 Subject: [PATCH 10/10] Re-add mildly useful SIGNATURES_SCHEMA --- securesystemslib/formats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/securesystemslib/formats.py b/securesystemslib/formats.py index b08e9853..10ade931 100755 --- a/securesystemslib/formats.py +++ b/securesystemslib/formats.py @@ -298,6 +298,8 @@ keyid = KEYID_SCHEMA, sig = HEX_SCHEMA) +SIGNATURES_SCHEMA = SCHEMA.ListOf(SIGNATURE_SCHEMA) + # A signable object. Holds the signing role and its associated signatures. SIGNABLE_SCHEMA = SCHEMA.Object( object_name = 'SIGNABLE_SCHEMA',