finish conversion to v2 format.

All the main functionality has been converted and tested,
but the Nosetests are still broken.

* fix #95: overhaul config settings & arguments
* fix #121: --user-filter regex matches entire user.
* fix #123: recognize v1 config files & err gracefully.
* change "org" to umapi
* fix write-stray-list (was not using new stray functions)
* fix add_umapi_user to do the right thing
* make sure we flush connections after adding users
* get rid of try_and_update_umapi_user - yay!
* update the release notes
* change the version to 2.0rc1

Author: adobeDan

RELEASE_NOTES.md

@@ -1,31 +1,42 @@
 # Release Notes for User Sync Tool Version 1.2
 
-These notes apply to 1.2rc1 of 2017-03-20.z
+These notes apply to 2.0rc1 of 2017-04-03.
+
+## New Arguments & Configuration Syntax
+
+There has been an extensive overhaul of both the configuration file
+syntax and the command-line argument syntax.  See
+[Issue 95](https://github.com/adobe-apiplatform/user-sync.py/issues/95)
+and the [docs](https://adobe-apiplatform.github.io/user-sync.py/)
+for details.
 
 ## New Features
 
 1. You can now exclude dashboard users from being updated or
-   deleted by User Sync. See the
-   [docs](https://adobe-apiplatform.github.io/user-sync.py/) for
-   details.
+deleted by User Sync. See the
+[docs](https://adobe-apiplatform.github.io/user-sync.py/) for
+details.
 2. There is more robust reporting for errors in configuration
-   files.
+files.
 3. The log now reports the User Sync version and gives the
-   details of how it was invoked.
+details of how it was invoked.
 4. You can now create and manage users of all identity types,
-   including Adobe IDs, both when operating from an LDAP
-   directory and from CSV files.
+including Adobe IDs, both when operating from an LDAP
+directory and from CSV files.
 5. You can now distinguish, when a customer directory user is
-   disabled or removed, whether to remove the matching Adobe-side
-   from the organization or also to delete his Adobe user
-   account.
+disabled or removed, whether to remove the matching Adobe-side
+user's product configurations and user groups, to remove the
+user but leave his cloud storage, or to delete his storage as well.
 
 ## Significant Bug Fixes
 
 1. There were many bugs fixed related to managing users of
-   identity types other than Federated ID.
+identity types other than Federated ID.
 2. There were many bugs fixes related to managing group
-   membership of all identity types.
+membership of all identity types.
+3. There was a complete overhaul of how users who have
+adobe group memberships in multiple organizations are
+managed.
 
 ## Changes in Behavior
 
@@ -34,5 +45,39 @@ some had applied only to Federated ID and some to Enterprise ID.
 
 ## Compatibility with Prior Versions
 
-Other than as noted above, existing configuration files and
-should work and have the same behavior.
+All existing configuration files, user input files,
+and command-line scripts will need to be revamped
+to be compatible with the new formats.  Here is a quick
+cheat sheet of what needs to be done:
+
+* replace `dashboard:` with `adobe_users:`
+* replace `directory:` with `directory_users:`
+* add a `connectors:` section under `adobe_users:` similar
+to the one under `directory_users`
+* change `owning` to be `umapi` and put it under `connectors`
+* if you access multiple organizations, remove
+`secondaries`, and put
+all the umapi specifications under `umapi` as a list,
+like this:
+```yaml
+adobe_users:
+  connectors:
+    umapi:
+      - primary-config.yml
+      - org1: org1-config.yml
+      - org2: org2-config.yml
+```
+* change `dashboard_groups` to `adobe_groups`
+* under `limits`, change `max_missing_users` to
+`max_adobe_only_users` and remove all other
+settings
+* if you have an extension, do the following:
+  * remove the per-context: user setting
+  * move all the settings under it to the top level in
+a new file, call it `extension.yaml`
+  * change `extensions` to `extension`, move it into
+the `directory_users` section, and put the relative
+path to the new `extension.yaml` file as its value.
+
+
+

tests/config_test.py

@@ -46,8 +46,8 @@ def test_get_dict_from_sources_str_found(self):
     @mock.patch('user_sync.identity_type.parse_identity_type')
     def test_get_rule_options(self, mock_id_type,mock_get_dict,mock_get_list,mock_get_string):
         mock_id_type.return_value = 'new_acc'
-        mock_get_dict.return_value = tests.helper.MockGetString()
-        mock_get_list.return_value = tests.helper.MockGetString()
+        mock_get_dict.return_value = tests.helper.MockDictConfig()
+        mock_get_list.return_value = tests.helper.MockDictConfig()
         options = self.conf_load.get_rule_options()
         expected = {
             'after_mapping_hook': None,

tests/helper.py

@@ -59,7 +59,8 @@ def assert_equal_users(unit_test, expected_users, actual_users):
     for expected_user in expected_users:
         actual_user = actual_users_by_email.get(expected_user['email'])
         unit_test.assertIsNotNone(expected_user)            
-        assert_equal_field_values(unit_test, expected_user, actual_user, ['firstname', 'lastname', 'email', 'country'])
+        assert_equal_field_values(unit_test, expected_user, actual_user,
+                                  ['firstname', 'lastname', 'email', 'country'])
         unit_test.assertSetEqual(set(expected_user['groups']), set(actual_user['groups']))
 
 def assert_equal_umapi_commands(unit_test, expected_commands, actual_commands):
@@ -75,7 +76,8 @@ def assert_equal_umapi_commands_list(unit_test, expected_commands_list, actual_c
         assert_equal_umapi_commands(unit_test, expected_commands, actual_commands)
 
 def create_umapi_commands(user, identity_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE):
-    commands = Commands(identity_type=identity_type, email=user['email'], username=user['username'], domain=user['domain'])
+    commands = Commands(identity_type=identity_type,
+                        email=user['email'], username=user['username'], domain=user['domain'])
     return commands
 
 def create_logger():
@@ -84,7 +86,7 @@ def create_logger():
 def create_action_manager():
     return ActionManager(None, "test org id", create_logger())
 
-class MockGetString():
+class MockDictConfig():
     def get_string(self,test1,test2):
         return 'test'
 

tests/rules_test.py

@@ -121,7 +121,7 @@ def _do_country_code_test(self, mock_umapi_commands, mock_connectors, identity_t
                        'email': '[email protected]',
                        'uid': '001'}
         }
-        mock_rules.add_umapi_user(user_key, mock_connectors)
+        mock_rules.add_umapi_user(user_key, set(), mock_connectors)
 
         if (identity_type == 'federatedID' and default_country_code == None and user_country_code == None):
             mock_rules.logger.error.assert_called_with('User %s cannot be added as it has a blank country code and no default has been specified.', user_key)

user_sync/app.py

@@ -183,10 +183,11 @@ def begin_work(config_loader):
             directory_connector_options['user_identity_type'] = rule_config['new_account_type']
         directory_connector.initialize(directory_connector_options)
 
-    umapi_primary_connector = user_sync.connector.umapi.UmapiConnector("primary", primary_umapi_config)
+    primary_name = '.primary' if secondary_umapi_configs else ''
+    umapi_primary_connector = user_sync.connector.umapi.UmapiConnector(primary_name, primary_umapi_config)
     umapi_other_connectors = {}
     for secondary_umapi_name, secondary_config in secondary_umapi_configs.iteritems():
-        umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector("secondary.%s" % secondary_umapi_name,
+        umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector(".secondary.%s" % secondary_umapi_name,
                                                                             secondary_config)
         umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector
     umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector, umapi_other_connectors)

user_sync/config.py

@@ -77,12 +77,28 @@ def get_logging_config(self):
         return self.main_config.get_dict_config('logging', True)
 
     def get_umapi_options(self):
+        '''
+        Read and return the primary and secondary umapi connector configs.
+        The primary is a singleton, the secondaries are a map from name to config.
+        The syntax in the config file is rather complex, which makes this code a bit complex;
+        be sure you read the detailed docs before trying to read this function.
+        We also check for and err out gracefully if it's a v1-style config file.
+        :return: tuple: (primary, secondary_map)
+        '''
+        if self.main_config.get_dict_config('dashboard', True):
+            raise AssertionException("Your main configuration file is still in v1 format.  Please convert it to v2.")
         adobe_users_config = self.main_config.get_dict_config('adobe_users', True)
-        connector_config = adobe_users_config and adobe_users_config.get_dict_config('connectors', True)
-        if connector_config:
-            umapi_config = connector_config.get_list('umapi', True)
+        if not adobe_users_config:
+            return {}, {}
+        connector_config = adobe_users_config.get_dict_config('connectors', True)
+        if not connector_config:
+            return {}, {}
+        umapi_config = connector_config.get_list('umapi', True)
+        if not umapi_config:
+            return {}, {}
         # umapi_config is a list of strings (primary umapi source files) followed by a
-        # list of dicts (secondary umapi source specifications, which are lists of strings)
+        # list of dicts (secondary umapi source specifications, whose keys are umapi names
+        # and whose values are a list of config file strings)
         secondary_config_sources = {}
         primary_config_sources = []
         for item in umapi_config:
@@ -140,7 +156,8 @@ def get_directory_groups(self):
         :rtype dict(str, list(user_sync.rules.AdobeGroup))
         '''
         adobe_groups_by_directory_group = {}
-        
+        if self.main_config.get_dict_config('directory', True):
+            raise AssertionException("Your main configuration file is still in v1 format.  Please convert it to v2.")
         groups_config = None
         directory_config = self.main_config.get_dict_config('directory_users', True)
         if (directory_config != None):
@@ -174,10 +191,10 @@ def get_directory_extension_options(self):
         if directory_config:
             sources = directory_config.get_list('extension', True)
             if sources:
-                options = self.get_dict_from_sources(directory_config and sources)
+                options = DictConfig('extension', self.get_dict_from_sources(sources))
                 if options:
-                    after_mapping_hook_text = options.get('after_mapping_hook')
-                    if not after_mapping_hook_text:
+                    after_mapping_hook_text = options.get_string('after_mapping_hook', True)
+                    if after_mapping_hook_text is None:
                         raise AssertionError("No after_mapping_hook found in extension configuration")
         return options
 
@@ -293,15 +310,15 @@ def get_rule_options(self):
         # now get the directory extension, if any
         after_mapping_hook = None
         extended_attributes = None
-        extension_options = self.get_directory_extension_options()
-        if extension_options:
-            after_mapping_hook_text = extension_options.get_string('after_mapping_hook')
+        extension_config = self.get_directory_extension_options()
+        if extension_config:
+            after_mapping_hook_text = extension_config.get_string('after_mapping_hook')
             after_mapping_hook = compile(after_mapping_hook_text, '<per-user after-mapping-hook>', 'exec')
-            extended_attributes = extension_options.get_list('extended_attributes')
+            extended_attributes = extension_config.get_list('extended_attributes')
             # declaration of extended adobe groups: this is needed for two reasons:
             # 1. it allows validation of group names, and matching them to adobe groups
             # 2. it allows removal of adobe groups not assigned by the hook
-            for extended_adobe_group in extension_options.get_list('extended_adobe_groups'):
+            for extended_adobe_group in extension_config.get_list('extended_adobe_groups'):
                 group = user_sync.rules.AdobeGroup.create(extended_adobe_group)
                 if group is None:
                     message = 'Extension contains illegal extended_adobe_group spec: ' + str(extended_adobe_group)

user_sync/connector/umapi.py

@@ -45,7 +45,7 @@ def __init__(self, name, caller_options):
         '''
         caller_config = user_sync.config.DictConfig('"%s umapi options"' % name, caller_options)
         builder = user_sync.config.OptionsBuilder(caller_config)
-        builder.set_string_value('logger_name', 'umapi.' + name)
+        builder.set_string_value('logger_name', 'umapi' + name)
         builder.set_bool_value('test_mode', False)
         options = builder.get_options()        
 
@@ -234,10 +234,16 @@ def __init__(self, connection, org_id, logger):
         :type org_id: str
         :type logger: logging.Logger
         '''
+        self.action_count = 0
+        self.error_count = 0
         self.items = []
         self.connection = connection
         self.org_id = org_id
         self.logger = logger.getChild('action')
+
+    def get_statistics(self):
+        '''Return the count of actions sent so far, and how many had errors.'''
+        return self.action_count, self.error_count
 
     def get_next_request_id(self):
         request_id = 'action_%d' % ActionManager.next_request_id
@@ -279,6 +285,7 @@ def add_action(self, action, callback = None):
             'callback': callback
         }
         self.items.append(item)
+        self.action_count += 1
         self.logger.log(logging.INFO, 'Added action: %s', json.dumps(action.wire_dict()))
         self._execute_action(action)
 
@@ -308,6 +315,7 @@ def process_sent_items(self, total_sent):
                 is_success = not action_errors or len(action_errors) == 0
 
                 if (not is_success):
+                    self.error_count += 1
                     for error in action_errors:
                         self.logger.error('Error in requestID: %s (User: %s, Command: %s): code: "%s" message: "%s"',
                                           action.frame.get("requestID"),