TDL-20088 updates to field, collection, and namespace restriction tes…

…ts (#108)

* TDL-20088 updates to field, collection, and namespace restriction tests

* Conditionally set expectations for fname restrictions based on db version

* Fix indentation bugs

* Attempt to address replication record count inconsistency dev-vm vs sandbox

* Review comments, split connection options apart, match image names and tags

* PR review comments, refactor object id collection, collection names, exception handling

* Refactor object id set unions with nested set comprehensions, update test-db to take image-tag at command line

* PR review comments, clean up test-db and fname test

bin/test-db

@@ -8,11 +8,16 @@ from argparse import RawTextHelpFormatter
 
 # singerio images have required keyfile
 image_name = "singerio/mongo"
-image_tag = "4.2-bionic"
-# image_tag = "4.4-bionic"
-# image_tag = "5.0"
 
-def start_container(name):
+# organize command options based on image_tag
+command_opts = {  # top level keys = supported image_tag versions, values = shell
+    '4.2-bionic': 'mongo',  # version 5.0.15 TODO Remove?
+    '4.4-bionic': 'mongo',  # version 4.4.6, also supports mongosh?
+    '5.0': 'mongosh',       # version 5.0.15, also supports mongo
+    '6.0': 'mongosh',       # version 6.0.4
+}
+
+def start_container(name, image_tag):
 
     START_COMMAND = """
     sudo docker run -e "MONGO_INITDB_ROOT_USERNAME={0}" -e "MONGO_INITDB_ROOT_PASSWORD={1}" \
@@ -40,8 +45,10 @@ def start_container(name):
     ip_addr = get_ip_addr(name)
     # If using image_version <=4.4.0-bionic use mongo, not mongosh
     CONFIGURE_COMMAND = """
-     docker exec mongo1 mongosh --host {} test -u {} -p {} --authenticationDatabase admin --eval {}
+     docker exec {} {} --host {} test -u {} -p {} --authenticationDatabase admin --eval {}
     """.format(
+        name,
+        command_opts[image_tag],
         ip_addr,
         os.getenv('TAP_MONGODB_USER'),
         os.getenv('TAP_MONGODB_PASSWORD'),
@@ -52,15 +59,13 @@ def start_container(name):
         sys.exit("Exited with code: {}, the docker command failed.".format(proc.returncode))
     print("Oplog configured correctly.")
 
-
 def get_ip_addr(name):
     IP_ADDR_COMMAND = "docker inspect {} | jq -r .[].NetworkSettings.IPAddress"
     print("Retrieving IP addr of mongodb container")
     ip_addr = subprocess.check_output(IP_ADDR_COMMAND.format(name), shell=True).decode('utf-8').rstrip()
     print(ip_addr)
     return ip_addr
 
-
 def stop_container(name):
     STOP_COMMAND = "sudo docker stop {0} && sudo docker rm {0}"
 
@@ -70,13 +75,17 @@ def stop_container(name):
         sys.exit("Exited with code: {}, the docker process failed to stop.".format(proc.returncode))
     print("Process stopped successfully")
 
-def connect_to_db(name):
-    CONNECT_COMMAND = "docker run -it --rm mongo mongo --host {} test -u {} -p {} --authenticationDatabase admin"
-
+def connect_to_db(name, image_tag):
+    CONNECT_COMMAND = "docker run -it --rm {}:{} {} --host {} test -u {} -p {} --authenticationDatabase admin"
     ip_addr = get_ip_addr(name)
 
     print("Attempting to connect to running container using a mongo container")
-    connect_command_format = CONNECT_COMMAND.format(ip_addr,
+    # Note: Shell is determined based on user provided image_tag, connect may fail if the shell
+    #       associated with the user provided image_tag is not supported by the running DB version.
+    connect_command_format = CONNECT_COMMAND.format(image_name,
+                                                    image_tag,
+                                                    command_opts[image_tag],
+                                                    ip_addr,
                                                     os.getenv('TAP_MONGODB_USER'),
                                                     os.getenv('TAP_MONGODB_PASSWORD'))
     print(connect_command_format)
@@ -97,16 +106,17 @@ Uses environment variables:
 parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=RawTextHelpFormatter)
 parser.add_argument('action', choices=['start','stop', 'connect'], help='action to perform with the container')
 parser.add_argument('--name', help="name assigned to running docker process", default='mongo1')
+parser.add_argument('--image-tag', choices=command_opts.keys(), help='Supported image tags, default=6.0', default='6.0')
 
 def main():
     parsed_args = parser.parse_args()
     # Potential arguments to add: pull, changing docker cointainer, changing password
     if parsed_args.action == 'start':
-        start_container(parsed_args.name)
+        start_container(parsed_args.name, parsed_args.image_tag)
     elif parsed_args.action == 'stop':
         stop_container(parsed_args.name)
     elif parsed_args.action == 'connect':
-        connect_to_db(parsed_args.name)
+        connect_to_db(parsed_args.name, parsed_args.image_tag)
 
 if __name__ == "__main__":
     main()

tests/test_mongodb_cname_restrictions.py

@@ -219,7 +219,8 @@ def test_run(self):
         messages_by_stream = runner.get_records_from_target_output()
         records_by_stream = {}
         for stream_name in self.expected_sync_streams():
-            records_by_stream[stream_name] = [x for x in messages_by_stream[stream_name]['messages'] if x.get('action') == 'upsert']
+            records_by_stream[stream_name] = [x for x in messages_by_stream[stream_name]['messages']
+                                              if x.get('action') == 'upsert']
 
         # assert that each of the streams that we synced are the ones that we expect to see
         record_count_by_stream = runner.examine_target_output_file(self,
@@ -233,9 +234,11 @@ def test_run(self):
             self.assertGreaterEqual(v, 6)
 
         # Verify that we got 2 records with _SDC_DELETED_AT
-        self.assertEqual(2, len([x['data'] for x in records_by_stream['1_simple_coll'] if x['data'].get('_sdc_deleted_at')]))
-        self.assertEqual(2, len([x['data'] for x in records_by_stream['_simple_coll_2'] if x['data'].get('_sdc_deleted_at')]))
+        for stream in self.expected_sync_streams():
+            self.assertEqual(2, len([x['data'] for x in records_by_stream[stream]
+                                     if x['data'].get('_sdc_deleted_at')]))
         # Verify that the _id of the records sent are the same set as the
         # _ids of the documents changed
-        actual = set([ObjectId(x['data']['_id']) for x in records_by_stream['1_simple_coll']]).union(set([ObjectId(x['data']['_id']) for x in records_by_stream['_simple_coll_2']]))
-        self.assertEqual(changed_ids, actual)
+        actual_ids = {ObjectId(x['data']['_id']) for stream in self.expected_sync_streams()
+                      for x in records_by_stream[stream]}
+        self.assertEqual(changed_ids, actual_ids)