Test performance and reliability fixes

tests/proxy_server.py

@@ -466,7 +466,7 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, prot
         port = int(sys.argv[1])
     else:
         port = 8080
-    server_address = ('127.0.0.1', port) # MODIFIED: changed from '::1'
+    server_address = ('localhost', port)
 
     # MODIFIED: Argument added, conditional below added to control INTERCEPT
     # setting.

tests/simple_server.py

@@ -35,7 +35,6 @@
 
 import sys
 import random
-import platform
 
 import six
 from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
@@ -80,6 +79,9 @@ def log_request(self, code='-', size='-'):
 else:
   handler = SimpleHTTPRequestHandler
 
+# Allow re-use so you can re-run tests as often as you want even if the
+# tests re-use ports. Otherwise TCP TIME-WAIT prevents reuse for ~1 minute
+six.moves.socketserver.TCPServer.allow_reuse_address = True
 httpd = six.moves.socketserver.TCPServer(('', PORT), handler)
 
 httpd.serve_forever()

tests/test_arbitrary_package_attack.py

@@ -39,12 +39,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf
@@ -55,6 +53,8 @@
 import tuf.client.updater as updater
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -87,9 +87,7 @@ def setUpClass(cls):
     logger.info('Serving on port: ' + str(cls.SERVER_PORT))
     cls.url = 'http://localhost:' + str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -106,6 +104,7 @@ def tearDownClass(cls):
     if cls.server_process.returncode is None:
       logger.info('Server process ' + str(cls.server_process.pid) + ' terminated.')
       cls.server_process.kill()
+      cls.server_process.wait()
 
 
 

tests/test_developer_tool.py

@@ -22,13 +22,10 @@
 """
 
 import os
-import time
-import datetime
 import unittest
 import logging
 import tempfile
 import shutil
-import unittest
 
 import tuf
 import tuf.log

tests/test_download.py

@@ -39,7 +39,6 @@
 import random
 import subprocess
 import sys
-import time
 import unittest
 
 import tuf
@@ -48,10 +47,11 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.exceptions
 
+import utils
+
 import requests.exceptions
 
 import securesystemslib
-import six
 
 logger = logging.getLogger(__name__)
 
@@ -81,13 +81,7 @@ def setUp(self):
     junk, rel_target_filepath = os.path.split(target_filepath)
     self.url = 'http://localhost:'+str(self.PORT)+'/'+rel_target_filepath
 
-    # Provide a delay long enough to allow the HTTPS servers to start.
-    # Encountered an error on one test system at delay value of 0.2s, so
-    # increasing to 0.5s.  Further increasing to 2s due to occasional failures
-    # in other tests in similar circumstances on AppVeyor.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(2)
+    utils.wait_for_server('localhost', self.PORT)
 
     # Computing hash of target file data.
     m = hashlib.md5()
@@ -102,6 +96,8 @@ def tearDown(self):
     if self.server_proc.returncode is None:
       logger.info('\tServer process '+str(self.server_proc.pid)+' terminated.')
       self.server_proc.kill()
+      # Drop return values of communicate()
+      self.server_proc.communicate()
     self.target_fileobj.close()
 
 
@@ -129,8 +125,8 @@ def test_download_url_to_tempfileobj_and_lengths(self):
     # the server-reported length of the file does not match the
     # required_length.  'updater.py' *does* verify the hashes of downloaded
     # content.
-    download.safe_download(self.url, self.target_data_length - 4)
-    download.unsafe_download(self.url, self.target_data_length - 4)
+    download.safe_download(self.url, self.target_data_length - 4).close()
+    download.unsafe_download(self.url, self.target_data_length - 4).close()
 
     # We catch 'tuf.exceptions.DownloadLengthMismatchError' for safe_download()
     # because it will not download more bytes than requested (in this case, a
@@ -140,7 +136,7 @@ def test_download_url_to_tempfileobj_and_lengths(self):
 
     # Calling unsafe_download() with a mismatched length should not raise an
     # exception.
-    download.unsafe_download(self.url, self.target_data_length + 1)
+    download.unsafe_download(self.url, self.target_data_length + 1).close()
 
 
 
@@ -277,12 +273,8 @@ def test_https_connection(self):
     expd_https_server_proc = popen_python(
         ['simple_https_server.py', port4, expired_cert_fname])
 
-    # Provide a delay long enough to allow the four HTTPS servers to start.
-    # Have encountered errors at 0.2s, 0.5s, and 2s, primarily on AppVeyor.
-    # Increasing to 4s for this test.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(3)
+    for port in range(self.PORT + 1, self.PORT + 5):
+      utils.wait_for_server('localhost', port)
 
     relative_target_fpath = os.path.basename(target_filepath)
     good_https_url = 'https://localhost:' + port1 + '/' + relative_target_fpath
@@ -314,13 +306,13 @@ def test_https_connection(self):
       # trusting the good certs (trusting the bad cert instead). Expect failure
       # because even though the server's cert file is otherwise OK, we don't
       # trust it.
-      print('Trying HTTPS download of target file: ' + good_https_url)
+      logger.info('Trying HTTPS download of target file: ' + good_https_url)
       with self.assertRaises(requests.exceptions.SSLError):
         download.safe_download(good_https_url, target_data_length)
       with self.assertRaises(requests.exceptions.SSLError):
         download.unsafe_download(good_https_url, target_data_length)
 
-      print('Trying HTTPS download of target file: ' + good2_https_url)
+      logger.info('Trying HTTPS download of target file: ' + good2_https_url)
       with self.assertRaises(requests.exceptions.SSLError):
         download.safe_download(good2_https_url, target_data_length)
       with self.assertRaises(requests.exceptions.SSLError):
@@ -372,6 +364,8 @@ def test_https_connection(self):
         if proc.returncode is None:
           logger.info('Terminating server process ' + str(proc.pid))
           proc.kill()
+          # drop return values
+          proc.communicate()
 
 
 

tests/test_endless_data_attack.py

@@ -42,12 +42,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf
@@ -57,6 +55,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.roledb
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +89,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.8)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -108,6 +106,7 @@ def tearDownClass(cls):
     if cls.server_process.returncode is None:
       logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
       cls.server_process.kill()
+      cls.server_process.wait()
 
 
 

tests/test_extraneous_dependencies_attack.py

@@ -45,12 +45,10 @@
 import os
 import tempfile
 import random
-import time
 import shutil
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf.formats
@@ -60,6 +58,8 @@
 import tuf.keydb
 import tuf.unittest_toolbox as unittest_toolbox
 
+import utils
+
 import securesystemslib
 import six
 
@@ -93,9 +93,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.7)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -112,6 +110,7 @@ def tearDownClass(cls):
     if cls.server_process.returncode is None:
       logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
       cls.server_process.kill()
+      cls.server_process.wait()
 
 
 

tests/test_indefinite_freeze_attack.py

@@ -52,7 +52,6 @@
 import json
 import subprocess
 import logging
-import sys
 import unittest
 
 import tuf.formats
@@ -64,6 +63,8 @@
 import tuf.keydb
 import tuf.exceptions
 
+import utils
+
 import securesystemslib
 import six
 
@@ -100,13 +101,7 @@ def setUpClass(cls):
     logger.info('Serving on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # Provide a delay long enough to allow the HTTP server to start.
-    # Encountered an error on one test system at delay value of 0.2s, so
-    # increasing to 0.5s.  Further increasing to 2s due to occasional failures
-    # in other tests in similar circumstances on AppVeyor.
-    # Expect to see "Connection refused" if this delay is not long enough
-    # (though other issues could cause that).
-    time.sleep(2)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -123,6 +118,7 @@ def tearDownClass(cls):
     if cls.server_process.returncode is None:
       logger.info('Server process '+str(cls.server_process.pid)+' terminated.')
       cls.server_process.kill()
+      cls.server_process.wait()
 
 
 

tests/test_key_revocation_integration.py

@@ -38,14 +38,11 @@
 from __future__ import unicode_literals
 
 import os
-import time
 import shutil
-import copy
 import tempfile
 import logging
 import random
 import subprocess
-import sys
 import unittest
 
 import tuf
@@ -56,6 +53,8 @@
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.client.updater as updater
 
+import utils
+
 import securesystemslib
 import six
 
@@ -89,9 +88,7 @@ def setUpClass(cls):
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    utils.wait_for_server('localhost', cls.SERVER_PORT)
 
 
 
@@ -108,6 +105,7 @@ def tearDownClass(cls):
     if cls.server_process.returncode is None:
       logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.')
       cls.server_process.kill()
+      cls.server_process.wait()
 
 
 

tests/test_mirrors.py

@@ -30,7 +30,6 @@
 
 import unittest
 
-import tuf
 import tuf.mirrors as mirrors
 import tuf.unittest_toolbox as unittest_toolbox