Merge pull request #212 from adobe-apiplatform/v2

prepare for 2.1.1rc1 build

Author: adobeDan

Makefile

@@ -1,11 +1,16 @@
 RM := rm -rf
+MKDIR := mkdir
 python_ldap_requirements := misc/build/python-ldap-requirements.txt
 
 ifeq ($(OS),Windows_NT)
 	output_file_extension = .pex
 	rm_path := $(shell python -c "import distutils.spawn; print(distutils.spawn.find_executable('rm'))")
 ifeq ($(rm_path),None)
 	RM := rmdir /S /Q
+endif
+	mkdir_path := $(shell python -c "import distutils.spawn; print(distutils.spawn.find_executable('mkdir'))")
+ifeq ($(mkdir_path),None)
+	MKDIR := md
 endif
 	python_arch := $(shell python -c "import platform; print platform.architecture()[0]")
 ifeq ($(python_arch),64bit)
@@ -19,7 +24,8 @@ output_filename = user-sync
 pex:
 	pip install --upgrade pip
 	pip install pex requests wheel
-	pip wheel -w wheelhouse -r misc/build/requirements.txt -r $(python_ldap_requirements)
+	pip wheel -w wheelhouse -r $(python_ldap_requirements)
+	-$(MKDIR) wheelhouse
 	-$(RM) $(output_dir)
 	pex \
 		-v -o $(output_dir)/$(output_filename)$(output_file_extension) -m user_sync.app \

RELEASE_NOTES.md

@@ -1,16 +1,21 @@
-# Release Notes for User Sync Tool Version 2.1
+# Release Notes for User Sync Tool Version 2.1.1
 
-These notes apply to v2.1 of 2017-05-12.
+These notes apply to v2.1.1rc1 of 2017-06-06.
 
 ## New Features
 
-1. We now have full Unicode support.  See [Issue 167](https://github.com/adobe-apiplatform/user-sync.py/issues/167) for details.
-2. We now support secure handling for all credential settings and credential files.  See [Issue 159](https://github.com/adobe-apiplatform/user-sync.py/issues/159) for design discussion, and read [the docs](https://adobe-apiplatform.github.io/user-sync.py/) for associated config changes.
+There are no new features in this release; bug fixes only.
 
 ## Bug Fixes
 
-Fixes for [Issue 181](https://github.com/adobe-apiplatform/user-sync.py/issues/181) and [Issue 189](https://github.com/adobe-apiplatform/user-sync.py/issues/189).
+There is one fix for some obscure Unicode edge cases (that were found only by code inspection): [Issue 167](https://github.com/adobe-apiplatform/user-sync.py/issues/167).
+
+User Sync no longer crashes if a user's LDAP email address is present but empty: [Issue 201](https://github.com/adobe-apiplatform/user-sync.py/issues/201).
+
+The proper packages were not present for secure credential storage on Linux platforms: [Issue 199](https://github.com/adobe-apiplatform/user-sync.py/issues/199).
+
+Still to come: a fix for secure key storage on Windows: [Issue 198](https://github.com/adobe-apiplatform/user-sync.py/issues/198).
 
 ## Compatibility with Prior Versions
 
-This version is fully backward-compatible with version 2.0.  There may be subtle behavioral changes due to bug fixes around support for non-Ascii characters.  There are also new configuration file options and a new command line argument that didn't exist in 2.0.
+This version is fully backward-compatible with version 2.1.

misc/build/python-ldap-requirements.txt

@@ -1 +0,0 @@
-python-ldap==2.4.25

misc/build/requirements.txt

@@ -18,6 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import sys
+
 from setuptools import setup
 
 version_namespace = {}
@@ -45,8 +47,17 @@
           'PyYAML',
           'umapi-client>=2.5',
           'psutil',
-          'keyring'
+          'keyring',
       ],
+      extras_require={
+          ':sys_platform=="linux" or sys_platform=="linux2"':[
+              'secretstorage',
+              'dbus-python'
+          ],
+          ':sys_platform=="win32"':[
+              'pywin32-ctypes==0.0.1'
+          ]
+      },
       setup_requires=['nose>=1.0'],
       tests_require=[
           'mock',

setup.py

@@ -29,6 +29,7 @@
 import user_sync.config
 import user_sync.connector.directory
 import user_sync.connector.umapi
+import user_sync.helper
 import user_sync.lockfile
 import user_sync.rules
 from user_sync.error import AssertionException
@@ -37,6 +38,21 @@
 LOG_STRING_FORMAT = '%(asctime)s %(process)d %(levelname)s %(name)s - %(message)s'
 LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
 
+# file global logger, defined early so later functions can refer to it.
+logger = logging.getLogger('main')
+
+
+def init_console_log():
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(logging.Formatter(LOG_STRING_FORMAT, LOG_DATE_FORMAT))
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(logging.DEBUG)
+    return handler
+
+# file global console_log_handler, defined early so later functions can refer to it.
+console_log_handler = init_console_log()
+
 
 def process_args():
     parser = argparse.ArgumentParser(description='User Sync from Adobe')
@@ -64,9 +80,9 @@ def process_args():
                              'side is updated to match.',
                         action='store_true', dest='update_user_info')
     parser.add_argument('--process-groups',
-                        help='if the membership in mapped groups differs between the enterprise directory and Adobe sides, '
+                        help='if membership in mapped groups differs between the enterprise directory and Adobe sides, '
                              'the group membership is updated on the Adobe side so that the memberships in mapped '
-                             'groups matches those on the enterprise directory side.',
+                             'groups match those on the enterprise directory side.',
                         action='store_true', dest='manage_groups')
     parser.add_argument('--adobe-only-user-action',
                         help="specify what action to take on Adobe users that don't match users from the "
@@ -94,19 +110,10 @@ def process_args():
     return parser.parse_args()
 
 
-def init_console_log():
-    console_log_handler = logging.StreamHandler(sys.stdout)
-    console_log_handler.setFormatter(logging.Formatter(LOG_STRING_FORMAT, LOG_DATE_FORMAT))
-    root_logger = logging.getLogger()
-    root_logger.addHandler(console_log_handler)
-    root_logger.setLevel(logging.DEBUG)
-    return console_log_handler
-
-
 def init_log(logging_config):
-    '''
+    """
     :type logging_config: user_sync.config.DictConfig
-    '''
+    """
     builder = user_sync.config.OptionsBuilder(logging_config)
     builder.set_bool_value('log_to_file', False)
     builder.set_string_value('file_log_directory', 'logs')
@@ -123,66 +130,66 @@ def init_log(logging_config):
     }
 
     console_log_level = level_lookup.get(options['console_log_level'])
-    if (console_log_level == None):
+    if console_log_level is None:
         console_log_level = logging.INFO
         logger.log(logging.WARNING, 'Unknown console log level: %s setting to info' % options['console_log_level'])
     console_log_handler.setLevel(console_log_level)
 
-    if options['log_to_file'] == True:
+    if options['log_to_file']:
         unknown_file_log_level = False
         file_log_level = level_lookup.get(options['file_log_level'])
-        if (file_log_level == None):
+        if file_log_level is None:
             file_log_level = logging.INFO
             unknown_file_log_level = True
         file_log_directory = options['file_log_directory']
         if not os.path.exists(file_log_directory):
             os.makedirs(file_log_directory)
 
         file_path = os.path.join(file_log_directory, datetime.date.today().isoformat() + ".log")
-        fileHandler = logging.FileHandler(file_path)
-        fileHandler.setLevel(file_log_level)
-        fileHandler.setFormatter(logging.Formatter(LOG_STRING_FORMAT, LOG_DATE_FORMAT))
-        logging.getLogger().addHandler(fileHandler)
+        file_handler = logging.FileHandler(file_path)
+        file_handler.setLevel(file_log_level)
+        file_handler.setFormatter(logging.Formatter(LOG_STRING_FORMAT, LOG_DATE_FORMAT))
+        logging.getLogger().addHandler(file_handler)
         if unknown_file_log_level:
             logger.log(logging.WARNING, 'Unknown file log level: %s setting to info' % options['file_log_level'])
 
 
 def begin_work(config_loader):
-    '''
+    """
     :type config_loader: user_sync.config.ConfigLoader
-    '''
+    """
 
     directory_groups = config_loader.get_directory_groups()
     primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options()
     rule_config = config_loader.get_rule_options()
 
     # process mapped configuration after the directory groups have been loaded, as mapped setting depends on this.
-    if (rule_config['directory_group_mapped']):
+    if rule_config['directory_group_mapped']:
         rule_config['directory_group_filter'] = set(directory_groups.iterkeys())
 
     # make sure that all the adobe groups are from known umapi connector names
     referenced_umapi_names = set()
     for groups in directory_groups.itervalues():
         for group in groups:
             umapi_name = group.umapi_name
-            if (umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME):
+            if umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME:
                 referenced_umapi_names.add(umapi_name)
     referenced_umapi_names.difference_update(secondary_umapi_configs.iterkeys())
 
-    if (len(referenced_umapi_names) > 0):
+    if len(referenced_umapi_names) > 0:
         raise AssertionException('Adobe groups reference unknown umapi connectors: %s' % referenced_umapi_names)
 
     directory_connector = None
     directory_connector_options = None
     directory_connector_module_name = config_loader.get_directory_connector_module_name()
-    if (directory_connector_module_name != None):
+    if directory_connector_module_name is not None:
         directory_connector_module = __import__(directory_connector_module_name, fromlist=[''])
         directory_connector = user_sync.connector.directory.DirectoryConnector(directory_connector_module)
         directory_connector_options = config_loader.get_directory_connector_options(directory_connector.name)
 
     config_loader.check_unused_config_keys()
 
-    if (directory_connector != None and directory_connector_options != None):
+    if directory_connector is not None and directory_connector_options is not None:
         # specify the default user_identity_type if it's not already specified in the options
         if 'user_identity_type' not in directory_connector_options:
             directory_connector_options['user_identity_type'] = rule_config['new_account_type']
@@ -198,7 +205,7 @@ def begin_work(config_loader):
     umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector, umapi_other_connectors)
 
     rule_processor = user_sync.rules.RuleProcessor(rule_config)
-    if (len(directory_groups) == 0 and rule_processor.will_manage_groups()):
+    if len(directory_groups) == 0 and rule_processor.will_manage_groups():
         logger.warn('no groups mapped in config file')
     rule_processor.run(directory_groups, directory_connector, umapi_connectors)
 
@@ -213,14 +220,14 @@ def create_config_loader(args):
 
 
 def create_config_loader_options(args):
-    '''
+    """
     This is where all the command-line arguments get set as options in the main config
     so that it appears as if they were loaded as part of the main configuration file.
     If you add an option that is supposed to be set from the command line here, you
     had better make sure you add it to the ones read in get_rule_options.
     :param args: the command-line args as parsed
     :return: the configured options for the config loader.
-    '''
+    """
     config_options = {
         'delete_strays': False,
         'directory_connector_module_name': None,
@@ -241,17 +248,17 @@ def create_config_loader_options(args):
     # --users
     users_args = args.users
     users_action = None if not users_args else user_sync.helper.normalize_string(users_args.pop(0))
-    if (users_action == None or users_action == 'all'):
+    if users_action is None or users_action == 'all':
         config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
-    elif (users_action == 'file'):
+    elif users_action == 'file':
         if len(users_args) == 0:
             raise AssertionException('Missing file path for --users %s [file_path]' % users_action)
         config_options['directory_connector_module_name'] = 'user_sync.connector.directory_csv'
         config_options['directory_connector_overridden_options'] = {'file_path': users_args.pop(0)}
-    elif (users_action == 'mapped'):
+    elif users_action == 'mapped':
         config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
         config_options['directory_group_mapped'] = True
-    elif (users_action == 'group'):
+    elif users_action == 'group':
         if len(users_args) == 0:
             raise AssertionException('Missing groups for --users %s [groups]' % users_action)
         config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
@@ -260,7 +267,7 @@ def create_config_loader_options(args):
         raise AssertionException('Unknown argument --users %s' % users_action)
 
     username_filter_pattern = args.username_filter_pattern
-    if (username_filter_pattern):
+    if username_filter_pattern:
         try:
             compiled_expression = re.compile(r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE)
         except Exception as e:
@@ -272,19 +279,19 @@ def create_config_loader_options(args):
     adobe_action_args = args.adobe_only_user_action
     if adobe_action_args is not None:
         adobe_action = None if not adobe_action_args else user_sync.helper.normalize_string(adobe_action_args.pop(0))
-        if (adobe_action == None or adobe_action == 'preserve'):
+        if adobe_action is None or adobe_action == 'preserve':
             pass  # no option settings needed
-        elif (adobe_action == 'exclude'):
+        elif adobe_action == 'exclude':
             config_options['exclude_strays'] = True
-        elif (adobe_action == 'write-file'):
+        elif adobe_action == 'write-file':
             if not adobe_action_args:
                 raise AssertionException('Missing file path for --adobe-only-user-action %s [file_path]' % adobe_action)
             config_options['stray_list_output_path'] = adobe_action_args.pop(0)
-        elif (adobe_action == 'delete'):
+        elif adobe_action == 'delete':
             config_options['delete_strays'] = True
-        elif (adobe_action == 'remove'):
+        elif adobe_action == 'remove':
             config_options['remove_strays'] = True
-        elif (adobe_action == 'remove-adobe-groups'):
+        elif adobe_action == 'remove-adobe-groups':
             config_options['disentitle_strays'] = True
         else:
             raise AssertionException('Unknown argument --adobe-only-user-action %s' % adobe_action)
@@ -305,11 +312,11 @@ def create_config_loader_options(args):
 
 
 def log_parameters(args):
-    '''
+    """
     Log the invocation parameters to make it easier to diagnose problem with customers
     :param args: namespace
     :return: None
-    '''
+    """
     logger.info('------- Invocation parameters -------')
     logger.info(' '.join(sys.argv))
     logger.debug('-------- Internal parameters --------')
@@ -349,7 +356,7 @@ def main():
             logger.critical("A different User Sync process is currently running.")
 
     except AssertionException as e:
-        if (not e.is_reported()):
+        if not e.is_reported():
             logger.critical(e.message)
             e.set_reported()
     except:
@@ -359,12 +366,9 @@ def main():
             pass
 
     finally:
-        if (run_stats != None):
+        if run_stats is not None:
             run_stats.log_end(logger)
 
 
-console_log_handler = init_console_log()
-logger = logging.getLogger('main')
-
 if __name__ == '__main__':
     main()