Merge branch 'development'

packages/python-google-compute-engine/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2013 Google Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py

@@ -28,6 +28,7 @@
 from google_compute_engine import logger
 from google_compute_engine import metadata_watcher
 from google_compute_engine.boto import boto_config
+from google_compute_engine.compat import distro_name
 from google_compute_engine.compat import urlerror
 from google_compute_engine.compat import urlrequest
 from google_compute_engine.instance_setup import instance_config
@@ -65,22 +66,38 @@ def __init__(self, debug=False):
       instance_config_metadata = self._GetInstanceConfig()
       self.instance_config = instance_config.InstanceConfig(
           logger=self.logger, instance_config_metadata=instance_config_metadata)
+
       if self.instance_config.GetOptionBool('InstanceSetup', 'set_host_keys'):
         host_key_types = self.instance_config.GetOptionString(
             'InstanceSetup', 'host_key_types')
         self._SetSshHostKeys(host_key_types=host_key_types)
+
       if self.instance_config.GetOptionBool('InstanceSetup', 'set_boto_config'):
         self._SetupBotoConfig()
+
+      self._DisableOvercommit()
+
     if self.instance_config.GetOptionBool(
         'InstanceSetup', 'optimize_local_ssd'):
       self._RunScript('google_optimize_local_ssd')
+
     if self.instance_config.GetOptionBool('InstanceSetup', 'set_multiqueue'):
       self._RunScript('google_set_multiqueue')
+
     try:
       self.instance_config.WriteConfig()
     except (IOError, OSError) as e:
       self.logger.warning(str(e))
 
+  def _DisableOvercommit(self, distro=distro_name):
+    """Disable overcommit accounting on E2 machine types."""
+
+    # Expected machine type format:
+    # 'projects/00000000000/machineTypes/n1-standard-1'
+    machine_type = self.metadata_dict['instance']['machineType'].split('/')[-1]
+    if machine_type.startswith('e2-') and 'bsd' not in distro:
+      subprocess.call(['sysctl', 'vm.overcommit_memory=1'])
+
   def _GetInstanceConfig(self):
     """Get the instance configuration specified in metadata.
 

packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py

@@ -46,7 +46,10 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config):
     mock_logger_instance = mock.Mock()
     mock_logger.Logger.return_value = mock_logger_instance
     mock_watcher_instance = mock.Mock()
-    mock_watcher_instance.GetMetadata.return_value = {'hello': 'world'}
+    mock_watcher_instance.GetMetadata.return_value = {
+        'hello': 'world',
+        'instance': {'machineType': 'fake'},
+    }
     mock_watcher.MetadataWatcher.return_value = mock_watcher_instance
     mock_config_instance = mock.Mock()
     mock_config_instance.GetOptionBool.return_value = True
@@ -79,6 +82,7 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config):
         mock.call.config.InstanceConfig().GetOptionBool(
             'InstanceSetup', 'set_boto_config'),
         mock.call.setup._SetupBotoConfig(),
+        mock.call.setup._DisableOvercommit(),
         # Setup for local SSD.
         mock.call.config.InstanceConfig().GetOptionBool(
             'InstanceSetup', 'optimize_local_ssd'),
@@ -91,7 +95,10 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config):
         mock.call.config.InstanceConfig().WriteConfig(),
     ]
     self.assertEqual(mocks.mock_calls, expected_calls)
-    self.assertEqual(mock_setup.metadata_dict, {'hello': 'world'})
+    self.assertEqual(mock_setup.metadata_dict, {
+        'hello': 'world',
+        'instance':{'machineType': 'fake'},
+    })
 
   @mock.patch('google_compute_engine.instance_setup.instance_setup.instance_config')
   @mock.patch('google_compute_engine.instance_setup.instance_setup.metadata_watcher')
@@ -421,6 +428,37 @@ def testSetupBotoConfigLocked(self, mock_boto):
     instance_setup.InstanceSetup._SetupBotoConfig(self.mock_setup)
     self.mock_logger.warning.assert_called_once_with('Test Error')
 
+  @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess')
+  def testDisableOvercommitNonE2(self, mock_subprocess):
+    self.mock_setup.metadata_dict = {
+        'instance': {
+            'machineType': 'projects/00000000000/machineTypes/n1-standard-1',
+        }
+    }
+    instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup)
+    mock_subprocess.call.assert_not_called()
+
+  @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess')
+  def testDisableOvercommitE2(self, mock_subprocess):
+    self.mock_setup.metadata_dict = {
+        'instance': {
+            'machineType': 'projects/00000000000/machineTypes/e2-standard-1',
+        }
+    }
+    instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup)
+    mock_subprocess.call.assert_called_once_with(
+        ['sysctl', 'vm.overcommit_memory=1'])
+
+  @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess')
+  def testDisableOvercommitBSD(self, mock_subprocess):
+    self.mock_setup.metadata_dict = {
+        'instance': {
+            'machineType': 'projects/00000000000/machineTypes/e2-standard-1',
+        }
+    }
+    instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup, 'bsd')
+    mock_subprocess.call.assert_not_called()
+
 
 if __name__ == '__main__':
   unittest.main()

packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py

@@ -97,7 +97,7 @@ def _DownloadAuthUrl(self, url, dest_dir):
 
     if not self.token:
       response = self.watcher.GetMetadata(
-          self.token_metadata_key, recursive=False, retry=False)
+          self.token_metadata_key, recursive=False, retry_limit=3)
 
       if not response:
         self.logger.info(

packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py

@@ -89,7 +89,7 @@ def testDownloadAuthUrlExceptionAndToken(
     # GetMetadata includes a prefix, so remove it.
     stripped_url = token_url.replace(metadata_prefix, '')
     mock_get_metadata.assert_called_once_with(
-        stripped_url, recursive=False, retry=False)
+        stripped_url, recursive=False, retry_limit=3)
 
     self.assertEqual(self.retriever.token, 'foo bar')
 
@@ -119,7 +119,7 @@ def testDownloadAuthUrlFallback(
     prefix = 'http://metadata.google.internal/computeMetadata/v1/'
     stripped_url = token_url.replace(prefix, '')
     mock_get_metadata.assert_called_once_with(
-        stripped_url, recursive=False, retry=False)
+        stripped_url, recursive=False, retry_limit=3)
     mock_download_url.assert_called_once_with(auth_url, self.dest_dir)
 
     self.assertIsNone(self.retriever.token)

packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py

@@ -154,33 +154,33 @@ def _GetMetadataUpdate(
 
   def _HandleMetadataUpdate(
       self, metadata_key='', recursive=True, wait=True, timeout=None,
-      retry=True):
+      retry_limit=None):
     """Wait for a successful metadata response.
 
     Args:
       metadata_key: string, the metadata key to watch for changes.
       recursive: bool, True if we should recursively watch for metadata changes.
       wait: bool, True if we should wait for a metadata change.
       timeout: int, timeout in seconds for returning metadata output.
-      retry: bool, True if we should retry on failure.
+      retry_limit: int or None, limit for number of times to retry on failure.
+          Retry indefinitely when set to None. Do not retry when set to zero.
 
     Returns:
       json, the deserialized contents of the metadata server.
     """
     exception = None
-    while True:
+    while retry_limit is None or retry_limit >= 0:
       try:
         return self._GetMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
             timeout=timeout)
       except (httpclient.HTTPException, socket.error, urlerror.URLError) as e:
+        if retry_limit is not None:
+          retry_limit -= 1
         if not isinstance(e, type(exception)):
           exception = e
           self.logger.error('GET request error retrieving metadata. %s.', e)
-        if retry:
-          continue
-        else:
-          break
+        time.sleep(1)
 
   def WatchMetadata(
       self, handler, metadata_key='', recursive=True, timeout=None):
@@ -202,18 +202,19 @@ def WatchMetadata(
         self.logger.exception('Exception calling the response handler. %s.', e)
 
   def GetMetadata(
-      self, metadata_key='', recursive=True, timeout=None, retry=True):
+      self, metadata_key='', recursive=True, timeout=None, retry_limit=None):
     """Retrieve the contents of metadata server for a metadata key.
 
     Args:
       metadata_key: string, the metadata key to watch for changes.
       recursive: bool, True if we should recursively watch for metadata changes.
       timeout: int, timeout in seconds for returning metadata output.
-      retry: bool, True if we should retry on failure.
+      retry_limit: int or None, limit for number of times to retry on failure.
+          Retry indefinitely when set to None. Do not retry when set to zero.
 
     Returns:
       json, the deserialized contents of the metadata server or None if error.
     """
     return self._HandleMetadataUpdate(
         metadata_key=metadata_key, recursive=recursive, wait=False,
-        timeout=timeout, retry=retry)
+        timeout=timeout, retry_limit=retry_limit)

packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py

@@ -259,12 +259,12 @@ def testHandleMetadataUpdateException(self):
     metadata_key = 'instance/id'
     recursive = False
     wait = False
-    retry = True
+    retry_limit = None
 
     self.assertEqual(
         self.mock_watcher._HandleMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
-            timeout=None, retry=retry),
+            timeout=None, retry_limit=retry_limit),
         {})
     expected_calls = [
         mock.call(
@@ -282,12 +282,12 @@ def testHandleMetadataUpdateExceptionNoRetry(self):
     metadata_key = 'instance/id'
     recursive = False
     wait = False
-    retry = False
+    retry_limit = 0
 
     self.assertIsNone(
         self.mock_watcher._HandleMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
-            timeout=None, retry=retry))
+            timeout=None, retry_limit=retry_limit))
     expected_calls = [
         mock.call(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
@@ -333,7 +333,8 @@ def testGetMetadata(self):
 
     self.assertEqual(self.mock_watcher.GetMetadata(), {})
     mock_response.assert_called_once_with(
-        metadata_key='', recursive=True, wait=False, timeout=None, retry=True)
+        metadata_key='', recursive=True, wait=False, timeout=None,
+        retry_limit=None)
     self.mock_watcher.logger.exception.assert_not_called()
 
   def testGetMetadataArgs(self):
@@ -342,15 +343,15 @@ def testGetMetadataArgs(self):
     self.mock_watcher._HandleMetadataUpdate = mock_response
     metadata_key = 'instance/id'
     recursive = False
-    retry = False
+    retry_limit = None
 
     response = self.mock_watcher.GetMetadata(
         metadata_key=metadata_key, recursive=recursive, timeout=60,
-        retry=retry)
+        retry_limit=retry_limit)
     self.assertEqual(response, {})
     mock_response.assert_called_once_with(
         metadata_key=metadata_key, recursive=False, wait=False, timeout=60,
-        retry=False)
+        retry_limit=None)
     self.mock_watcher.logger.exception.assert_not_called()
 
 

packages/python-google-compute-engine/packaging/debian/changelog

@@ -1,3 +1,10 @@
+python-google-compute-engine (1:20191210.00-g1) stable; urgency=medium
+
+  * Enable sysctl change for E2 platform.
+  * Support retry limits in metadata retrieval.
+
+ -- Google Cloud Team <gc-team@google.com>  Tue, 10 Dec 2019 15:02:59 -0800
+
 python-google-compute-engine (1:20191120.00-g1) stable; urgency=medium
 
   * REVERT: Retry metadata lookups in agent.

packages/python-google-compute-engine/packaging/setup_deb.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="python-google-compute-engine"
-VERSION="20191120.00"
+VERSION="20191210.00"
 
 working_dir=${PWD}
 if [[ $(basename "$working_dir") != $NAME ]]; then

packages/python-google-compute-engine/packaging/setup_rpm.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="python-google-compute-engine"
-VERSION="20191120.00"
+VERSION="20191210.00"
 
 rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION}
 working_dir=${PWD}

packages/python-google-compute-engine/setup.py

@@ -37,7 +37,7 @@
     name='google-compute-engine',
     packages=setuptools.find_packages(),
     url='https://github.com/GoogleCloudPlatform/compute-image-packages',
-    version='20191112.0',
+    version='20191210.0',
     # Entry points create scripts in /usr/bin that call a function.
     entry_points={
         'console_scripts': [