Merge branch 'main_issues_77_and_171' into main

tests/assets/create_from_files_test/create.yml

@@ -9,3 +9,4 @@ models:
  - 25: ['tif', 'tiff', 'jp2', 'png', 'gif', 'jpg', 'jpeg']
  - 22: ['mp3', 'wav', 'aac']
  - 26: ['mp4']
+log_json: true

tests/assets/create_from_files_test/create_drupal_8.yml

@@ -0,0 +1,13 @@
+task: create_from_files
+host: "http://localhost:8000"
+username: admin
+password: islandora
+input_dir: tests/assets/create_from_files_test/files
+models:
+ - 23: ['zip', 'tar', '']
+ - 27: ['pdf', 'doc', 'docx', 'ppt', 'pptx']
+ - 25: ['tif', 'tiff', 'jp2', 'png', 'gif', 'jpg', 'jpeg']
+ - 22: ['mp3', 'wav', 'aac']
+ - 26: ['mp4']
+log_json: true
+drupal_8: true

tests/assets/create_paged_content_from_directories_test/books_drupal_8.yml

@@ -0,0 +1,9 @@
+paged_content_from_directories: true
+paged_content_page_model_tid: 29
+task: create
+host: "http://localhost:8000"
+username: admin
+password: islandora
+input_dir: tests/assets/create_paged_content_from_directories_test/samplebooks
+media_type: image
+drupal_8: true

tests/islandora_tests.py

@@ -345,6 +345,43 @@ def tearDown(self):
             os.remove(self.rollback_file_path)
 
 
+class TestCreateFromFilesDrupal8(unittest.TestCase):
+
+    def setUp(self):
+        self.current_dir = os.path.dirname(os.path.abspath(__file__))
+        create_config_file_path = os.path.join(self.current_dir, 'assets', 'create_from_files_test', 'create_drupal_8.yml')
+        self.create_cmd = ["./workbench", "--config", create_config_file_path]
+
+        self.temp_dir = tempfile.gettempdir()
+        self.nid_file = os.path.join(self.temp_dir, 'workbenchcreatefromfilestestnids.txt')
+
+    def test_create_from_files_drupal_8(self):
+        nids = list()
+        create_output = subprocess.check_output(self.create_cmd)
+        create_output = create_output.decode().strip()
+        create_lines = create_output.splitlines()
+        with open(self.nid_file, "a") as fh:
+            fh.write("node_id\n")
+            for line in create_lines:
+                if 'created at' in line:
+                    nid = line.rsplit('/', 1)[-1]
+                    nid = nid.strip('.')
+                    nids.append(nid)
+                    fh.write(nid + "\n")
+
+        self.assertEqual(len(nids), 3)
+
+    def tearDown(self):
+        delete_config_file_path = os.path.join(self.current_dir, 'assets', 'create_from_files_test', 'delete.yml')
+        delete_cmd = ["./workbench", "--config", delete_config_file_path]
+        delete_output = subprocess.check_output(delete_cmd)
+        os.remove(self.nid_file)
+
+        self.rollback_file_path = os.path.join(self.current_dir, 'assets', 'create_from_files_test', 'files', 'rollback.csv')
+        if os.path.exists(self.rollback_file_path):
+            os.remove(self.rollback_file_path)
+
+
 class TestCreateWithNewTypedRelation(unittest.TestCase):
     # Note: You can't run this test class on its own, e.g.,
     # python3 tests/islandora_tests.py TestCreateWithNewTypedRelation.
@@ -588,6 +625,81 @@ def tearDown(self):
             os.remove(rollback_file_path)
 
 
+class TestCreatePagedContentFromDirectoriesDrupal8(unittest.TestCase):
+
+    def setUp(self):
+        self.current_dir = os.path.dirname(os.path.abspath(__file__))
+        create_config_file_path = os.path.join(self.current_dir, 'assets', 'create_paged_content_from_directories_test', 'books_drupal_8.yml')
+
+        yaml = YAML()
+        with open(create_config_file_path, 'r') as f:
+            config_file_contents = f.read()
+        config_data = yaml.load(config_file_contents)
+        config = {}
+        for k, v in config_data.items():
+            config[k] = v
+        self.islandora_host = config['host']
+
+        self.create_cmd = ["./workbench", "--config", create_config_file_path]
+
+        self.temp_dir = tempfile.gettempdir()
+        self.nid_file = os.path.join(self.temp_dir, 'workbenchcreatepagedcontentfromdirectoriestestnids.txt')
+
+    def test_create_paged_content_from_directories_drupal_8(self):
+        nids = list()
+        create_output = subprocess.check_output(self.create_cmd)
+        create_output = create_output.decode().strip()
+
+        # Write a file to the system's temp directory containing the node IDs of the
+        # nodes created during this test so they can be deleted in tearDown().
+        create_lines = create_output.splitlines()
+        with open(self.nid_file, "a") as fh:
+            fh.write("node_id\n")
+            for line in create_lines:
+                if 'created at' in line:
+                    nid = line.rsplit('/', 1)[-1]
+                    nid = nid.strip('.')
+                    nids.append(nid)
+                    fh.write(nid + "\n")
+
+        self.assertEqual(len(nids), 4)
+
+        # Test a page object's 'field_member_of' value to see if it matches its
+        # parent's node ID. In this test, we'll test the second page. Note: the
+        # metadata CSV file used to create the paged content and page objects
+        # uses hard-coded term IDs from the Islandora Models taxonomy as used
+        # in the Islandora Playbook. If they change or are different in the
+        # Islandora this test is running against, this test will fail. Also note
+        # that this test creates media and does not delete them.
+        parent_node_id_to_test = nids[0]
+        child_node_id_to_test = nids[2]
+        node_url = self.islandora_host + '/node/' + child_node_id_to_test + '?_format=json'
+        response = requests.get(node_url)
+        node_json = json.loads(response.text)
+        field_member_of = node_json['field_member_of'][0]['target_id']
+
+        self.assertEqual(int(parent_node_id_to_test), field_member_of)
+
+        # Test that the 'field_weight' value of the second node is 2.
+        self.assertEqual(2, node_json['field_weight'][0]['value'])
+
+    def tearDown(self):
+        delete_config_file_path = os.path.join(self.current_dir, 'assets', 'create_paged_content_from_directories_test', 'delete.yml')
+        delete_cmd = ["./workbench", "--config", delete_config_file_path]
+        delete_output = subprocess.check_output(delete_cmd)
+        delete_output = delete_output.decode().strip()
+        delete_lines = delete_output.splitlines()
+        os.remove(self.nid_file)
+
+        preprocessed_csv_path = os.path.join(self.current_dir, 'assets', 'create_paged_content_from_directories_test', 'samplebooks', 'metadata.csv.prepocessed')
+        if os.path.exists(preprocessed_csv_path):
+            os.remove(preprocessed_csv_path)
+
+        rollback_file_path = os.path.join(self.current_dir, 'assets', 'create_paged_content_from_directories_test', 'samplebooks', 'rollback.csv')
+        if os.path.exists(rollback_file_path):
+            os.remove(rollback_file_path)
+
+
 class TestTaxonomies (unittest.TestCase):
 
     def setUp(self):
@@ -740,17 +852,23 @@ def test_create_with_non_latin_text(self):
 
         self.assertEqual(len(nids), 3)
 
-        url = self.islandora_host + '/solr-search/content?search_api_fulltext=一九二四年六月十二日'
+        url = self.islandora_host + '/node/' + str(nids[0]) + '?_format=json'
         response = requests.get(url)
-        self.assertIn('一九二四年六月十二日', response.text)
+        node = json.loads(response.text)
+        title = str(node['title'][0]['value'])
+        self.assertEqual(title, '一九二四年六月十二日')
 
-        url = self.islandora_host + '/solr-search/content?search_api_fulltext=ᐊᑕᐅᓯᖅ ᓄᓇ, ᐅᓄᖅᑐᑦ ᓂᐲᑦ'
+        url = self.islandora_host + '/node/' + str(nids[1]) + '?_format=json'
         response = requests.get(url)
-        self.assertIn('ᐊᑕᐅᓯᖅ ᓄᓇ, ᐅᓄᖅᑐᑦ ᓂᐲᑦ', response.text)
+        node = json.loads(response.text)
+        title = str(node['title'][0]['value'])
+        self.assertEqual(title, 'सरकारी दस्तावेज़')
 
-        url = self.islandora_host + '/solr-search/content?search_api_fulltext=सरकारी दस्तावेज़'
+        url = self.islandora_host + '/node/' + str(nids[2]) + '?_format=json'
         response = requests.get(url)
-        self.assertIn('सरकारी दस्तावेज़', response.text)
+        node = json.loads(response.text)
+        title = str(node['title'][0]['value'])
+        self.assertEqual(title, 'ᐊᑕᐅᓯᖅ ᓄᓇ, ᐅᓄᖅᑐᑦ ᓂᐲᑦ')
 
     def tearDown(self):
         delete_config_file_path = os.path.join(self.current_dir, 'assets', 'non_latin_text_test', 'delete.yml')

tests/unit_tests.py

@@ -302,5 +302,24 @@ def tearDown(self):
         os.remove(self.csv_file_path)
 
 
+class TestDrupalCoreVersionNumbers(unittest.TestCase):
+    def test_version_numbers(self):
+        lower_versions = ['9.0', '9.1', '8.10.0-dev', '9.0.0-dev']
+        for version in lower_versions:
+            version_number = workbench_utils.convert_drupal_core_version_to_number(version)
+            res = version_number < tuple([9, 2])
+            self.assertTrue(res)
+
+        version_number = workbench_utils.convert_drupal_core_version_to_number('9.2')
+        self.assertTrue(version_number == tuple([9, 2]))
+
+        # higher_versions = ['9.3', '9.10.0-dev', '9.20.0-dev']
+        higher_versions = ['9.3', '9.10.0-dev']
+        for version in higher_versions:
+            version_number = workbench_utils.convert_drupal_core_version_to_number(version)
+            res = version_number > tuple([9, 2])
+            self.assertTrue(res)
+
+
 if __name__ == '__main__':
     unittest.main()

workbench

@@ -110,14 +110,17 @@ def create():
             if custom_field == 'url_alias':
                 continue
 
+            # 'media_use_tid' is a reserved CSV field.
+            if custom_field == 'media_use_tid':
+                continue
+
             # Execute field preprocessor scripts, if any are configured. Note that these scripts
             # are applied to the entire value from the CSV field and not split field values,
             # e.g., if a field is multivalued, the preprocesor must split it and then reassemble
             # it back into a string before returning it. Note that preprocessor scripts work only
             # on string data and not on binary data like images, etc. and only on custom fields
             # (so not title).
-            if 'preprocessors' in config and len(
-                    config['preprocessors']) > 0:
+            if 'preprocessors' in config and len(config['preprocessors']) > 0:
                 for field, command in config['preprocessors'].items():
                     if field in csv_column_headers:
                         output, return_code = preprocess_field_data(
@@ -261,12 +264,12 @@ def create():
         node_response = issue_request(config, 'POST', node_endpoint, node_headers, node, None)
         if node_response.status_code == 201:
             node_uri = node_response.headers['location']
+            returned_node = json.loads(node_response.text)
 
             # If Pathauto URL alias creation for nodes is enabled, the location header
             # returns the alias, not the /node/xxx URL, which includes the node ID. In
             # this case, get the node ID from the response body.
             if not re.match(r'/node/\d+$', node_uri):
-                returned_node = json.loads(node_response.text)
                 node_id = returned_node['nid'][0]['value']
                 node_uri = config['host'] + '/node/' + str(node_id)
 
@@ -312,7 +315,10 @@ def create():
         if node_response.status_code == 201:
             # If there is something in the 'file' field, create the media from it.
             if config['nodes_only'] is False and 'file' in row and len(row['file']) != 0:
-                media_response_status_code = create_media(config, row['file'], node_uri, row_for_media)
+                if drupal_8 is True:
+                    media_response_status_code = create_islandora_media(config, row['file'], node_uri, row_for_media)
+                else:
+                    media_response_status_code = create_media(config, row['file'], node_nid, row_for_media)
                 allowed_media_response_codes = [201, 204]
                 if media_response_status_code in allowed_media_response_codes:
                     if config['progress_bar'] is False:
@@ -699,8 +705,10 @@ def add_media():
         node_uri = config['host'] + '/node/' + row['node_id']
         node_response = issue_request(config, 'GET', node_json_url)
         if node_response.status_code == 200:
-            media_response_status_code = create_media(
-                config, row['file'], node_uri, None)
+            if drupal_8 is True:
+                media_response_status_code = create_islandora_media(config, row['file'], node_uri, None)
+            else:
+                media_response_status_code = create_media(config, row['file'], row['node_id'], None)
             allowed_media_response_codes = [201, 204]
             if media_response_status_code in allowed_media_response_codes:
                 if config['progress_bar'] is False:
@@ -825,19 +833,17 @@ def create_from_files():
             file_path = os.path.join(config['input_dir'], file_name)
             media_type = set_media_type(file_path, config)
             fake_csv_record = collections.OrderedDict()
+            fake_csv_record['title'] = filename_without_extension
             if media_type == 'image':
                 fake_csv_record['image_alt_text'] = filename_without_extension
-            media_response_status_code = create_media(
-                config, file_name, node_uri, fake_csv_record)
+            if drupal_8 is True:
+                media_response_status_code = create_islandora_media(config, file_name, node_uri, fake_csv_record)
+            else:
+                media_response_status_code = create_media(config, file_name, node_nid, fake_csv_record)
             allowed_media_response_codes = [201, 204]
             if media_response_status_code in allowed_media_response_codes:
                 if config['progress_bar'] is False:
-                    print(
-                        '+ ' +
-                        media_type.title() +
-                        " media for " +
-                        filename_without_extension +
-                        " created.")
+                    print('+ ' + media_type.title() + " media for " + filename_without_extension + " created.")
                 logging.info("Media for %s created.", file_path)
         else:
             logging.error(
@@ -879,7 +885,12 @@ if config['task'] != 'create_from_files' and config['input_csv'].endswith('.xlsx
 
 ping_islandora(config, print_message=True)
 
-# Apparently, there's no builtin way of getting the number of items in a
+if config['nodes_only'] is False:
+    check_drupal_core_version(config)
+
+drupal_8 = set_drupal_8(config)
+
+# Apparently, there's no built-in way of getting the number of items in a
 # DictReader, so we read the CSV file, convert it to a list, and get its length.
 if config['progress_bar'] is True:
     if config['task'] != 'create_from_files':