Merge pull request #9 from balihoo/ENG-595-add-deps-upgrade

ENG-595 | Add support in manifest to upgrade dependencies

blambda/utils/env_manager.py

@@ -27,6 +27,7 @@
 
 DEFAULT_RUNTIME = py36
 
+
 class EnvManager(object):
     def __init__(self, runtime):
         super(EnvManager, self).__init__()
@@ -83,16 +84,17 @@ def create_virtualenv(self, clean=True):
         sp.check_call(tools_upgrade + ['setuptools'])
         sp.check_call(tools_upgrade + ['pip'])
 
-    def install_dependencies(self, lib_dir, **dependencies):
+    def install_dependencies(self, lib_dir, segregated_targets, **dependencies):
         """ Install dependencies with pip
 
         Args:
             lib_dir: directory where the dependencies should be stored (this is separate from the virtualenv dir)
+            segregated_targets: separate target lib for dependencies with conflicting namespaces
             **dependencies: dict of { 'dependency_name': 'dependency_version' }
 
         """
         with concurrent.futures.ThreadPoolExecutor(max_workers=32) as parallel:
-            all_futures = {parallel.submit(self._install_dependency, dep, lib_dir, version): dep
+            all_futures = {parallel.submit(self._install_dependency, dep, lib_dir, version, segregated_targets): dep
                            for dep, version in dependencies.items()}  # key is the futures object, value is the dep
 
             for future in concurrent.futures.as_completed(all_futures):
@@ -104,8 +106,13 @@ def install_dependencies(self, lib_dir, **dependencies):
                     sys.exit(1)
                 else:
                     cprint('pip: finished installing ' + dep, 'blue')
+                    # Below code will execute only for libs with conflicting namespaces
+                    if dep in segregated_targets.keys():
+                        cprint('Running rsync: to transfer data from ' + str(segregated_targets[dep]) + ' to ' + str(lib_dir), 'blue')
+                        sp.check_call(["rsync", "-rv", "-P", f"{segregated_targets[dep]}/", f"{lib_dir}/"])
+                        sp.check_call(["rm", "-rf", f"{segregated_targets[dep]}"])
 
-    def _install_dependency(self, dep, lib_dir, version):
+    def _install_dependency(self, dep, lib_dir, version, segregated_targets):
         install_cmd = ['install']
         local = dep.startswith('/home/')
         linked = version == "link"
@@ -122,12 +129,15 @@ def _install_dependency(self, dep, lib_dir, version):
             # there's an outstanding bug which prevents pip from using both -e and -t at the same time
             install_cmd.extend(('-e', dep))
         else:
+
             if version:
                 if dep.startswith("git+"):
                     dep += "@" + version
                 else:
+                    if dep in segregated_targets.keys():
+                        lib_dir = segregated_targets[dep]
                     dep += "==" + version
             install_cmd.extend([dep, '-t', lib_dir])
 
         # actually install the package using pip
-        sp.check_call([self.pip] + install_cmd)
+        sp.check_call([self.pip] + install_cmd)

blambda/utils/lambda_manifest.py

@@ -110,6 +110,9 @@ def deployed_name(self):
     def runtime(self):
         return self.json.get('options', {}).get('Runtime', 'python2.7').lower()
 
+    def segregated_lib_dir(self, path):
+        return self.basedir / ('lib_' + path)
+
     def source_files(self, dest_dir: Path = None):
         """ Return a generator yielding tuples of (source_file, destination_target), unraveling any globs along the way
 
@@ -170,6 +173,7 @@ def process_manifest(self, clean=False, prod=False):
         for command in manifest.get('before setup', []):
             spawn(command, show=True, working_directory=self.basedir, raise_on_fail=True)
 
+        segregated_targets = manifest.get('segregated_targets', [])
         dependencies = manifest.get('dependencies', {})
         if not prod:
             dev_deps = manifest.get('dev dependencies', {})
@@ -184,7 +188,8 @@ def process_manifest(self, clean=False, prod=False):
             if clean and os.path.exists(self.lib_dir):
                 cprint(f"clean install -- removing {self.lib_dir}", 'yellow')
                 shutil.rmtree(self.lib_dir)
-            env.install_dependencies(self.lib_dir, **deps_to_install)
+            segregated_targets = {k: self.segregated_lib_dir(k) for k in segregated_targets}
+            env.install_dependencies(self.lib_dir, segregated_targets, **deps_to_install)
 
         elif 'node' in self.runtime:
             # currently there's no way to npm install to a directory other than <whatever>/node_modules
@@ -261,4 +266,4 @@ def validate_dependencies(dependencies):
         cprint("\nExact version required for: {}\n  --> run 'update_versions' "
                "to set the latest version for all dependencies\n".format(", ".join(bad_version)),
                'red')
-        raise Exception('Failed to install dependencies')
+        raise Exception('Failed to install dependencies')