Fix error "[Errno 111] Connection refused" and make logs more usable

tests/simple_server.py

@@ -66,9 +66,16 @@ def log_request(self, code='-', size='-'):
 
 # NOTE: On Windows/Python2 tests that use this simple_server.py in a
 # subprocesses hang after a certain amount of requests (~68), if a PIPE is
-# passed as Popen's stderr argument. As a simple workaround we silence the
-# server on those Windows/Py2 to not fill the buffer.
-if six.PY2 and platform.system() == 'Windows':
+# passed as Popen's stderr argument. This problem doesn't emerge if
+# we silence the HTTP messages.
+# If you decide to receive the HTTP messages, then this bug
+# could reappear.
+use_quiet_http_request_handler = True
+
+if len(sys.argv) > 2:
+ use_quiet_http_request_handler = sys.argv[2]
+
+if use_quiet_http_request_handler:
  handler = QuietHTTPRequestHandler
 else:
  handler = SimpleHTTPRequestHandler

tests/test_arbitrary_package_attack.py

@@ -81,7 +81,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: ' + str(cls.server_process.pid))
  logger.info('Serving on port: ' + str(cls.SERVER_PORT))

tests/test_endless_data_attack.py

@@ -83,7 +83,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: '+str(cls.server_process.pid))
  logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_extraneous_dependencies_attack.py

@@ -87,7 +87,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: '+str(cls.server_process.pid))
  logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_indefinite_freeze_attack.py

@@ -94,7 +94,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: '+str(cls.server_process.pid))
  logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_key_revocation_integration.py

@@ -83,7 +83,7 @@ def setUpClass(cls):
  # key files, etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('\n\tServer process started.')
  logger.info('\tServer process id: '+str(cls.server_process.pid))
  logger.info('\tServing on port: '+str(cls.SERVER_PORT))

tests/test_mix_and_match_attack.py

@@ -88,7 +88,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: '+str(cls.server_process.pid))
  logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_multiple_repositories_integration.py

@@ -128,17 +128,22 @@ def setUp(self):
  while self.SERVER_PORT == self.SERVER_PORT2:
  self.SERVER_PORT2 = random.SystemRandom().randint(30000, 45000)
 
- command = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT)]
- command2 = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT2)]
+ # Needed because in some tests simple_server.py cannot be found.
+ # The reason is that the current working directory
+ # has been changed when executing a subprocess.
+ SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
 
- self.server_process = subprocess.Popen(command, stderr=subprocess.PIPE,
+ command = ['python', SIMPLE_SERVER_PATH, str(self.SERVER_PORT)]
+ command2 = ['python', SIMPLE_SERVER_PATH, str(self.SERVER_PORT2)]
+
+ self.server_process = subprocess.Popen(command,
  cwd=self.repository_directory)
 
  logger.debug('Server process started.')
  logger.debug('Server process id: ' + str(self.server_process.pid))
  logger.debug('Serving on port: ' + str(self.SERVER_PORT))
 
- self.server_process2 = subprocess.Popen(command2, stderr=subprocess.PIPE,
+ self.server_process2 = subprocess.Popen(command2,
  cwd=self.repository_directory2)
 
 

tests/test_replay_attack.py

@@ -87,7 +87,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('Server process started.')
  logger.info('Server process id: '+str(cls.server_process.pid))
  logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_slow_retrieval_attack.py

@@ -103,7 +103,7 @@ def _start_slow_server(self, mode):
  # as a delegated role 'targets/role1', three target files, five key files,
  # etc.
  command = ['python', 'slow_retrieval_server.py', str(self.SERVER_PORT), mode]
- server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ server_process = subprocess.Popen(command)
  logger.info('Slow Retrieval Server process started.')
  logger.info('Server process id: '+str(server_process.pid))
  logger.info('Serving on port: '+str(self.SERVER_PORT))

tests/test_updater.py

@@ -90,6 +90,11 @@ def setUpClass(cls):
  # temporary files are always removed, even when exceptions occur.
  cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd())
 
+ # Needed because in some tests simple_server.py cannot be found.
+ # The reason is that the current working directory
+ # has been changed when executing a subprocess.
+ cls.SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
+
  # Launch a SimpleHTTPServer (serves files in the current directory).
  # Test cases will request metadata and target files that have been
  # pre-generated in 'tuf/tests/repository_data', which will be served
@@ -98,8 +103,8 @@ def setUpClass(cls):
  # as a delegated role 'targets/role1', three target files, five key files,
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
- command = ['python', '-m', 'tests.simple_server', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ command = ['python', cls.SIMPLE_SERVER_PATH, str(cls.SERVER_PORT)]
+ cls.server_process = subprocess.Popen(command)
  logger.info('\n\tServer process started.')
  logger.info('\tServer process id: '+str(cls.server_process.pid))
  logger.info('\tServing on port: '+str(cls.SERVER_PORT))
@@ -1091,8 +1096,8 @@ def test_6_get_one_valid_targetinfo(self):
  # The SimpleHTTPServer started in the setupclass has a tendency to
  # timeout in Windows after a few tests.
  SERVER_PORT = random.randint(30000, 45000)
- command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
- server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+ server_process = subprocess.Popen(command)
 
  # NOTE: Following error is raised if a delay is not long enough:
  # <urlopen error [Errno 111] Connection refused>
@@ -1359,8 +1364,8 @@ def test_7_updated_targets(self):
  # The SimpleHTTPServer started in the setupclass has a tendency to
  # timeout in Windows after a few tests.
  SERVER_PORT = random.randint(30000, 45000)
- command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
- server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+ server_process = subprocess.Popen(command)
 
  # NOTE: Following error is raised if a delay is not long enough to allow
  # the server process to set up and start listening:
@@ -1491,8 +1496,8 @@ def test_8_remove_obsolete_targets(self):
  # The SimpleHTTPServer started in the setupclass has a tendency to
  # timeout in Windows after a few tests.
  SERVER_PORT = random.randint(30000, 45000)
- command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
- server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+ server_process = subprocess.Popen(command)
 
  # NOTE: Following error is raised if a delay is not long enough to allow
  # the server process to set up and start listening:
@@ -1824,6 +1829,11 @@ def setUp(self):
  self.temporary_repository_root = self.make_temp_directory(directory=
  self.temporary_directory)
 
+ # Needed because in some tests simple_server.py cannot be found.
+ # The reason is that the current working directory
+ # has been changed when executing a subprocess.
+ self.SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
+
  # The original repository, keystore, and client directories will be copied
  # for each test case.
  original_repository = os.path.join(original_repository_files, 'repository')
@@ -1875,17 +1885,17 @@ def setUp(self):
  self.SERVER_PORT = 30001
  self.SERVER_PORT2 = 30002
 
- command = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT)]
- command2 = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT2)]
+ command = ['python', self.SIMPLE_SERVER_PATH, str(self.SERVER_PORT)]
+ command2 = ['python', self.SIMPLE_SERVER_PATH, str(self.SERVER_PORT2)]
 
- self.server_process = subprocess.Popen(command, stderr=subprocess.PIPE,
+ self.server_process = subprocess.Popen(command,
  cwd=self.repository_directory)
 
  logger.debug('Server process started.')
  logger.debug('Server process id: ' + str(self.server_process.pid))
  logger.debug('Serving on port: ' + str(self.SERVER_PORT))
 
- self.server_process2 = subprocess.Popen(command2, stderr=subprocess.PIPE,
+ self.server_process2 = subprocess.Popen(command2,
  cwd=self.repository_directory2)
 
  logger.debug('Server process 2 started.')

tests/test_updater_root_rotation_integration.py

@@ -90,7 +90,7 @@ def setUpClass(cls):
  # etc.
  cls.SERVER_PORT = random.randint(30000, 45000)
  command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
- cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+ cls.server_process = subprocess.Popen(command)
  logger.info('\n\tServer process started.')
  logger.info('\tServer process id: '+str(cls.server_process.pid))
  logger.info('\tServing on port: '+str(cls.SERVER_PORT))