Test performance and reliability fixes

tests/proxy_server.py

@@ -466,7 +466,7 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, prot
  port = int(sys.argv[1])
  else:
  port = 8080
- server_address = ('127.0.0.1', port) # MODIFIED: changed from '::1'
+ server_address = ('localhost', port)
 
  # MODIFIED: Argument added, conditional below added to control INTERCEPT
  # setting.

tests/simple_server.py

@@ -35,7 +35,6 @@
 
 import sys
 import random
-import platform
 
 import six
 from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
@@ -80,6 +79,9 @@ def log_request(self, code='-', size='-'):
 else:
  handler = SimpleHTTPRequestHandler
 
+# Allow re-use so you can re-run tests as often as you want even if the
+# tests re-use ports. Otherwise TCP TIME-WAIT prevents reuse for ~1 minute
+six.moves.socketserver.TCPServer.allow_reuse_address = True
 httpd = six.moves.socketserver.TCPServer(('', PORT), handler)
 
 httpd.serve_forever()

tests/test_arbitrary_package_attack.py

@@ -39,12 +39,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf
@@ -55,6 +53,8 @@
 import tuf.client.updater as updater
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -87,9 +87,7 @@ def setUpClass(cls):
  logger.info('Serving on port: ' + str(cls.SERVER_PORT))
  cls.url = 'http://localhost:' + str(cls.SERVER_PORT) + os.path.sep
 
- # NOTE: Following error is raised if a delay is not applied:
- # <urlopen error [Errno 111] Connection refused>
- time.sleep(1)
+ utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -106,6 +104,7 @@ def tearDownClass(cls):
  if cls.server_process.returncode is None:
  logger.info('Server process ' + str(cls.server_process.pid) + ' terminated.')
  cls.server_process.kill()
+ cls.server_process.wait()
 
 
 

tests/test_developer_tool.py

@@ -22,13 +22,10 @@
 """
 
 import os
-import time
-import datetime
 import unittest
 import logging
 import tempfile
 import shutil
-import unittest
 
 import tuf
 import tuf.log

tests/test_download.py

@@ -39,7 +39,6 @@
 import random
 import subprocess
 import sys
-import time
 import unittest
 
 import tuf
@@ -48,10 +47,11 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.exceptions
 
+import utils
+
 import requests.exceptions
 
 import securesystemslib
-import six
 
 logger = logging.getLogger(__name__)
 
@@ -81,13 +81,7 @@ def setUp(self):
  junk, rel_target_filepath = os.path.split(target_filepath)
  self.url = 'http://localhost:'+str(self.PORT)+'/'+rel_target_filepath
 
- # Provide a delay long enough to allow the HTTPS servers to start.
- # Encountered an error on one test system at delay value of 0.2s, so
- # increasing to 0.5s. Further increasing to 2s due to occasional failures
- # in other tests in similar circumstances on AppVeyor.
- # Expect to see "Connection refused" if this delay is not long enough
- # (though other issues could cause that).
- time.sleep(2)
+ utils.wait_for_server('localhost', self.PORT)
 
  # Computing hash of target file data.
  m = hashlib.md5()
@@ -102,6 +96,8 @@ def tearDown(self):
  if self.server_proc.returncode is None:
  logger.info('\tServer process '+str(self.server_proc.pid)+' terminated.')
  self.server_proc.kill()
+ # Drop return values of communicate()
+ self.server_proc.communicate()
  self.target_fileobj.close()
 
 
@@ -129,8 +125,8 @@ def test_download_url_to_tempfileobj_and_lengths(self):
  # the server-reported length of the file does not match the
  # required_length. 'updater.py' *does* verify the hashes of downloaded
  # content.
- download.safe_download(self.url, self.target_data_length - 4)
- download.unsafe_download(self.url, self.target_data_length - 4)
+ download.safe_download(self.url, self.target_data_length - 4).close()
+ download.unsafe_download(self.url, self.target_data_length - 4).close()
 
  # We catch 'tuf.exceptions.DownloadLengthMismatchError' for safe_download()
  # because it will not download more bytes than requested (in this case, a
@@ -140,7 +136,7 @@ def test_download_url_to_tempfileobj_and_lengths(self):
 
  # Calling unsafe_download() with a mismatched length should not raise an
  # exception.
- download.unsafe_download(self.url, self.target_data_length + 1)
+ download.unsafe_download(self.url, self.target_data_length + 1).close()
 
 
 
@@ -277,12 +273,8 @@ def test_https_connection(self):
  expd_https_server_proc = popen_python(
  ['simple_https_server.py', port4, expired_cert_fname])
 
- # Provide a delay long enough to allow the four HTTPS servers to start.
- # Have encountered errors at 0.2s, 0.5s, and 2s, primarily on AppVeyor.
- # Increasing to 4s for this test.
- # Expect to see "Connection refused" if this delay is not long enough
- # (though other issues could cause that).
- time.sleep(3)
+ for port in range(self.PORT + 1, self.PORT + 5):
+ utils.wait_for_server('localhost', port)
 
  relative_target_fpath = os.path.basename(target_filepath)
  good_https_url = 'https://localhost:' + port1 + '/' + relative_target_fpath
@@ -314,13 +306,13 @@ def test_https_connection(self):
  # trusting the good certs (trusting the bad cert instead). Expect failure
  # because even though the server's cert file is otherwise OK, we don't
  # trust it.
- print('Trying HTTPS download of target file: ' + good_https_url)
+ logger.info('Trying HTTPS download of target file: ' + good_https_url)
  with self.assertRaises(requests.exceptions.SSLError):
  download.safe_download(good_https_url, target_data_length)
  with self.assertRaises(requests.exceptions.SSLError):
  download.unsafe_download(good_https_url, target_data_length)
 
- print('Trying HTTPS download of target file: ' + good2_https_url)
+ logger.info('Trying HTTPS download of target file: ' + good2_https_url)
  with self.assertRaises(requests.exceptions.SSLError):
  download.safe_download(good2_https_url, target_data_length)
  with self.assertRaises(requests.exceptions.SSLError):
@@ -372,6 +364,8 @@ def test_https_connection(self):
  if proc.returncode is None:
  logger.info('Terminating server process ' + str(proc.pid))
  proc.kill()
+ # drop return values
+ proc.communicate()
 
 
 

tests/test_endless_data_attack.py

@@ -42,12 +42,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf
@@ -57,6 +55,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.roledb
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +89,7 @@ def setUpClass(cls):
  logger.info('Serving on port: '+str(cls.SERVER_PORT))
  cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
- # NOTE: Following error is raised if a delay is not applied:
- # <urlopen error [Errno 111] Connection refused>
- time.sleep(.8)
+ utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -108,6 +106,7 @@ def tearDownClass(cls):
  if cls.server_process.returncode is None:
  logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
  cls.server_process.kill()
+ cls.server_process.wait()
 
 
 

tests/test_extraneous_dependencies_attack.py

@@ -45,12 +45,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf.formats
@@ -60,6 +58,8 @@
 import tuf.keydb
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -93,9 +93,7 @@ def setUpClass(cls):
  logger.info('Serving on port: '+str(cls.SERVER_PORT))
  cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
- # NOTE: Following error is raised if a delay is not applied:
- # <urlopen error [Errno 111] Connection refused>
- time.sleep(.7)
+ utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -112,6 +110,7 @@ def tearDownClass(cls):
  if cls.server_process.returncode is None:
  logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
  cls.server_process.kill()
+ cls.server_process.wait()
 
 
 

tests/test_indefinite_freeze_attack.py

@@ -52,7 +52,6 @@
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf.formats
@@ -64,6 +63,8 @@
 import tuf.keydb
 import tuf.exceptions
 
+import utils
+
 import securesystemslib
 import six
 
@@ -100,13 +101,7 @@ def setUpClass(cls):
  logger.info('Serving on port: '+str(cls.SERVER_PORT))
  cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
- # Provide a delay long enough to allow the HTTP server to start.
- # Encountered an error on one test system at delay value of 0.2s, so
- # increasing to 0.5s. Further increasing to 2s due to occasional failures
- # in other tests in similar circumstances on AppVeyor.
- # Expect to see "Connection refused" if this delay is not long enough
- # (though other issues could cause that).
- time.sleep(2)
+ utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -123,6 +118,7 @@ def tearDownClass(cls):
  if cls.server_process.returncode is None:
  logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
  cls.server_process.kill()
+ cls.server_process.wait()
 
 
 

tests/test_key_revocation_integration.py

@@ -38,14 +38,11 @@
 from __future__ import unicode_literals
 
 import os
-import time
 import shutil
-import copy
 import tempfile
 import logging
 import random
 import subprocess
-import sys
 import unittest
 
 import tuf
@@ -56,6 +53,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.client.updater as updater
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +88,7 @@ def setUpClass(cls):
  logger.info('\tServing on port: '+str(cls.SERVER_PORT))
  cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
- # NOTE: Following error is raised if a delay is not applied:
- # <urlopen error [Errno 111] Connection refused>
- time.sleep(1)
+ utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -108,6 +105,7 @@ def tearDownClass(cls):
  if cls.server_process.returncode is None:
  logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.')
  cls.server_process.kill()
+ cls.server_process.wait()
 
 
 

tests/test_mirrors.py

@@ -30,7 +30,6 @@
 
 import unittest
 
-import tuf
 import tuf.mirrors as mirrors
 import tuf.unittest_toolbox as unittest_toolbox