Separate targets and snapshot/timestamp schemas

This separation and refactoring is part of the change to
make length and hashes optional for timestamp and snapshot roles.

It separates FILEINFO_SCHEMA into two separate schemas:
TARGETS_FILEINFO_SCHEMA and METADATA_FILEINFO_SCHEMA.
The distinction is needed because as of version 1.0.1 of the tuf
spec targets role has mandatory length and hashes, and
snapshot and timestamp roles have a mandatory version, and optional
length and hashes.
That's why targets can't share the same schemas
as timestamp and snapshot.

Because of that schema distinction, make_fileinfo had to be too
separated into make_targets_fileinfo and make_metadata_fileinfo.

Signed-off-by: Martin Vrachev <mvrachev@vmware.com>

tests/test_arbitrary_package_attack.py

@@ -180,7 +180,7 @@ def test_without_tuf(self):
  client_target_path = os.path.join(self.client_directory, 'file1.txt')
  self.assertFalse(os.path.exists(client_target_path))
  length, hashes = securesystemslib.util.get_file_details(target_path)
- fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  url_prefix = self.repository_mirrors['mirror1']['url_prefix']
  url_file = os.path.join(url_prefix, 'targets', 'file1.txt')
@@ -190,20 +190,20 @@ def test_without_tuf(self):
 
  self.assertTrue(os.path.exists(client_target_path))
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
  self.assertEqual(fileinfo, download_fileinfo)
 
  # Test: Download a target file that has been modified by an attacker.
  with open(target_path, 'wt') as file_object:
  file_object.write('add malicious content.')
  length, hashes = securesystemslib.util.get_file_details(target_path)
- malicious_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ malicious_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # On Windows, the URL portion should not contain back slashes.
  six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path)
 
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is unequal to the original trusted version.
  self.assertNotEqual(download_fileinfo, fileinfo)

tests/test_endless_data_attack.py

@@ -184,7 +184,7 @@ def test_without_tuf(self):
  client_target_path = os.path.join(self.client_directory, 'file1.txt')
  self.assertFalse(os.path.exists(client_target_path))
  length, hashes = securesystemslib.util.get_file_details(target_path)
- fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  url_prefix = self.repository_mirrors['mirror1']['url_prefix']
  url_file = os.path.join(url_prefix, 'targets', 'file1.txt')
@@ -194,15 +194,15 @@ def test_without_tuf(self):
 
  self.assertTrue(os.path.exists(client_target_path))
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
  self.assertEqual(fileinfo, download_fileinfo)
 
  # Test: Download a target file that has been modified by an attacker with
  # extra data.
  with open(target_path, 'a') as file_object:
  file_object.write('append large amount of data' * 100000)
  large_length, hashes = securesystemslib.util.get_file_details(target_path)
- malicious_fileinfo = tuf.formats.make_fileinfo(large_length, hashes)
+ malicious_fileinfo = tuf.formats.make_targets_fileinfo(large_length, hashes)
 
  # Is the modified file actually larger?
  self.assertTrue(large_length > length)
@@ -211,7 +211,7 @@ def test_without_tuf(self):
  six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path)
 
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is unequal to the original trusted version.
  self.assertNotEqual(download_fileinfo, fileinfo)
@@ -235,10 +235,10 @@ def test_with_tuf(self):
  # Verify the client's downloaded file matches the repository's.
  target_path = os.path.join(self.repository_directory, 'targets', 'file1.txt')
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
  self.assertEqual(fileinfo, download_fileinfo)
 
  # Modify 'file1.txt' and confirm that the TUF client only downloads up to
@@ -257,7 +257,7 @@ def test_with_tuf(self):
  # extra data appended should be discarded by the client, so the downloaded
  # file size and hash should not have changed.
  length, hashes = securesystemslib.util.get_file_details(client_target_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
  self.assertEqual(fileinfo, download_fileinfo)
 
  # Test that the TUF client does not download large metadata files, as well.

tests/test_formats.py

@@ -119,11 +119,14 @@ def test_schemas(self):
  'keyval': {'public': 'pubkey',
  'private': 'privkey'}}),
 
- 'FILEINFO_SCHEMA': (tuf.formats.FILEINFO_SCHEMA,
+ 'TARGETS_FILEINFO_SCHEMA': (tuf.formats.TARGETS_FILEINFO_SCHEMA,
  {'length': 1024,
  'hashes': {'sha256': 'A4582BCF323BCEF'},
  'custom': {'type': 'paintjob'}}),
 
+ 'METADATA_FILEINFO_SCHEMA': (tuf.formats.METADATA_FILEINFO_SCHEMA,
+ {'version': 1}),
+
  'FILEDICT_SCHEMA': (tuf.formats.FILEDICT_SCHEMA,
  {'metadata/root.json': {'length': 1024,
  'hashes': {'sha256': 'ABCD123'},
@@ -241,7 +244,8 @@ def test_schemas(self):
  'version': 8,
  'expires': '1985-10-21T13:20:00Z',
  'meta': {'metadattimestamp.json': {'length': 1024,
- 'hashes': {'sha256': 'AB1245'}}}}),
+ 'hashes': {'sha256': 'AB1245'},
+ 'version': 1}}}),
 
  'MIRROR_SCHEMA': (tuf.formats.MIRROR_SCHEMA,
  {'url_prefix': 'http://localhost:8001',
@@ -394,7 +398,7 @@ def test_build_dict_conforming_to_schema(self):
  length = 88
  hashes = {'sha256': '3c7fe3eeded4a34'}
  expires = '1985-10-21T13:20:00Z'
- filedict = {'snapshot.json': {'length': length, 'hashes': hashes}}
+ filedict = {'snapshot.json': {'length': length, 'hashes': hashes, 'version': 1}}
 
 
  # Try with and without _type and spec_version, both of which are
@@ -797,28 +801,60 @@ def test_make_signable(self):
 
 
 
- def test_make_fileinfo(self):
+ def test_make_targets_fileinfo(self):
  # Test conditions for valid arguments.
  length = 1024
  hashes = {'sha256': 'A4582BCF323BCEF', 'sha512': 'A4582BCF323BFEF'}
- version = 8
  custom = {'type': 'paintjob'}
 
- FILEINFO_SCHEMA = tuf.formats.FILEINFO_SCHEMA
- make_fileinfo = tuf.formats.make_fileinfo
- self.assertTrue(FILEINFO_SCHEMA.matches(make_fileinfo(length, hashes, version, custom)))
- self.assertTrue(FILEINFO_SCHEMA.matches(make_fileinfo(length, hashes)))
+ TARGETS_FILEINFO_SCHEMA = tuf.formats.TARGETS_FILEINFO_SCHEMA
+ make_targets_fileinfo = tuf.formats.make_targets_fileinfo
+ self.assertTrue(TARGETS_FILEINFO_SCHEMA.matches(make_targets_fileinfo(length, hashes, custom)))
+ self.assertTrue(TARGETS_FILEINFO_SCHEMA.matches(make_targets_fileinfo(length, hashes)))
 
  # Test conditions for invalid arguments.
  bad_length = 'bad'
  bad_hashes = 'bad'
  bad_custom = 'bad'
 
- self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, bad_length, hashes, custom)
- self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, bad_hashes, custom)
- self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, hashes, bad_custom)
- self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, bad_length, hashes)
- self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, bad_hashes)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+ bad_length, hashes, custom)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+ length, bad_hashes, custom)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+ length, hashes, bad_custom)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+ bad_length, hashes)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+ length, bad_hashes)
+
+
+
+ def test_make_metadata_fileinfo(self):
+ # Test conditions for valid arguments.
+ length = 1024
+ hashes = {'sha256': 'A4582BCF323BCEF', 'sha512': 'A4582BCF323BFEF'}
+ version = 8
+
+ METADATA_FILEINFO_SCHEMA = tuf.formats.METADATA_FILEINFO_SCHEMA
+ make_metadata_fileinfo = tuf.formats.make_metadata_fileinfo
+ self.assertTrue(METADATA_FILEINFO_SCHEMA.matches(make_metadata_fileinfo(
+ version, length, hashes)))
+ self.assertTrue(METADATA_FILEINFO_SCHEMA.matches(make_metadata_fileinfo(version)))
+
+ # Test conditions for invalid arguments.
+ bad_version = 'bad'
+ bad_length = 'bad'
+ bad_hashes = 'bad'
+
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+ bad_version, length, hashes)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+ version, bad_length, hashes)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+ version, length, bad_hashes)
+ self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+ bad_version)
 
 
 

tests/test_indefinite_freeze_attack.py

@@ -230,15 +230,15 @@ def test_without_tuf(self):
  shutil.copy(timestamp_path, client_timestamp_path)
 
  length, hashes = securesystemslib.util.get_file_details(timestamp_path)
- fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  url_prefix = self.repository_mirrors['mirror1']['url_prefix']
  url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json')
 
  six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
  length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is equal to the current local file.
  self.assertEqual(download_fileinfo, fileinfo)

tests/test_replay_attack.py

@@ -205,7 +205,7 @@ def test_without_tuf(self):
  # The fileinfo of the previous version is saved to verify that it is indeed
  # accepted by the non-TUF client.
  length, hashes = securesystemslib.util.get_file_details(backup_timestamp)
- previous_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ previous_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Modify the timestamp file on the remote repository.
  repository = repo_tool.load_repository(self.repository_directory)
@@ -227,7 +227,7 @@ def test_without_tuf(self):
  # Save the fileinfo of the new version generated to verify that it is
  # saved by the client.
  length, hashes = securesystemslib.util.get_file_details(timestamp_path)
- new_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ new_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  url_prefix = self.repository_mirrors['mirror1']['url_prefix']
  url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json')
@@ -238,7 +238,7 @@ def test_without_tuf(self):
  six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
  length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is equal to the new version.
  self.assertEqual(download_fileinfo, new_fileinfo)
@@ -251,7 +251,7 @@ def test_without_tuf(self):
  six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
  length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is equal to the previous version.
  self.assertEqual(download_fileinfo, previous_fileinfo)
@@ -278,7 +278,7 @@ def test_with_tuf(self):
  # The fileinfo of the previous version is saved to verify that it is indeed
  # accepted by the non-TUF client.
  length, hashes = securesystemslib.util.get_file_details(backup_timestamp)
- previous_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ previous_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Modify the timestamp file on the remote repository.
  repository = repo_tool.load_repository(self.repository_directory)
@@ -300,7 +300,7 @@ def test_with_tuf(self):
  # Save the fileinfo of the new version generated to verify that it is
  # saved by the client.
  length, hashes = securesystemslib.util.get_file_details(timestamp_path)
- new_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ new_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Refresh top-level metadata, including 'timestamp.json'. Installation of
  # new version of 'timestamp.json' is expected.
@@ -309,7 +309,7 @@ def test_with_tuf(self):
  client_timestamp_path = os.path.join(self.client_directory,
  self.repository_name, 'metadata', 'current', 'timestamp.json')
  length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
- download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+ download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
  # Verify 'download_fileinfo' is equal to the new version.
  self.assertEqual(download_fileinfo, new_fileinfo)

tests/test_repository_lib.py

@@ -227,7 +227,7 @@ def test_get_metadata_filenames(self):
 
 
 
- def test_get_metadata_fileinfo(self):
+ def test_get_targets_metadata_fileinfo(self):
  # Test normal case.
  temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory)
  test_filepath = os.path.join(temporary_directory, 'file.txt')
@@ -236,31 +236,31 @@ def test_get_metadata_fileinfo(self):
  file_object.write('test file')
 
  # Generate test fileinfo object. It is assumed SHA256 and SHA512 hashes
- # are computed by get_metadata_fileinfo().
+ # are computed by get_targets_metadata_fileinfo().
  file_length = os.path.getsize(test_filepath)
  sha256_digest_object = securesystemslib.hash.digest_filename(test_filepath)
  sha512_digest_object = securesystemslib.hash.digest_filename(test_filepath, algorithm='sha512')
  file_hashes = {'sha256': sha256_digest_object.hexdigest(),
  'sha512': sha512_digest_object.hexdigest()}
  fileinfo = {'length': file_length, 'hashes': file_hashes}
- self.assertTrue(tuf.formats.FILEINFO_SCHEMA.matches(fileinfo))
+ self.assertTrue(tuf.formats.TARGETS_FILEINFO_SCHEMA.matches(fileinfo))
 
  storage_backend = securesystemslib.storage.FilesystemBackend()
 
- self.assertEqual(fileinfo, repo_lib.get_metadata_fileinfo(test_filepath,
+ self.assertEqual(fileinfo, repo_lib.get_targets_metadata_fileinfo(test_filepath,
  storage_backend))
 
 
  # Test improperly formatted argument.
  self.assertRaises(securesystemslib.exceptions.FormatError,
- repo_lib.get_metadata_fileinfo, 3,
+ repo_lib.get_targets_metadata_fileinfo, 3,
  storage_backend)
 
 
  # Test non-existent file.
  nonexistent_filepath = os.path.join(temporary_directory, 'oops.txt')
  self.assertRaises(securesystemslib.exceptions.Error,
- repo_lib.get_metadata_fileinfo,
+ repo_lib.get_targets_metadata_fileinfo,
  nonexistent_filepath, storage_backend)
 
 
@@ -589,7 +589,7 @@ def test_generate_snapshot_metadata_without_hashes_and_length(self):
  if metadata_filename not in \
  ['root.json', 'targets.json', 'timestamp.json', 'snapshot.json']:
 
- # Check that both length and hashes are not is not calculated
+ # Check that both length and hashes are not are not calculated
  self.assertNotIn('length', metadata_files_info_dict[metadata_filename])
  self.assertNotIn('hashes', metadata_files_info_dict[metadata_filename])
 

tests/test_repository_tool.py

@@ -460,8 +460,8 @@ def test_writeall_no_files(self):
  # Add target fileinfo
  target1_hashes = {'sha256': 'c2986576f5fdfd43944e2b19e775453b96748ec4fe2638a6d2f32f1310967095'}
  target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
- target1_fileinfo = tuf.formats.make_fileinfo(555, target1_hashes)
- target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+ target1_fileinfo = tuf.formats.make_targets_fileinfo(555, target1_hashes)
+ target2_fileinfo = tuf.formats.make_targets_fileinfo(37, target2_hashes)
  target1 = 'file1.txt'
  target2 = 'file2.txt'
  repository.targets.add_target(target1, fileinfo=target1_fileinfo)
@@ -1527,7 +1527,7 @@ def test_add_target_to_bin(self):
 
  # Test adding a target with fileinfo
  target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
- target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+ target2_fileinfo = tuf.formats.make_targets_fileinfo(37, target2_hashes)
  target2_filepath = 'file2.txt'
 
  self.targets_object.add_target_to_bin(target2_filepath, 16, fileinfo=target2_fileinfo)