Add Dataset integration tests - Tables, Folders

tests_new/integration_tests/modules/s3_datasets/global_conftest.py

@@ -117,6 +117,36 @@ def session_s3_dataset1(client1, group1, org1, session_env1, session_id, testdat
             delete_s3_dataset(client1, session_env1['environmentUri'], ds)
 
 
+
+@pytest.fixture(scope='session')
+def session_s3_dataset2_with_table(client1, group1, org1, session_env1, session_id, testdata):
+    ds = None
+    try:
+        ds = create_s3_dataset(
+            client1,
+            owner='someone',
+            group=group1,
+            org_uri=org1['organizationUri'],
+            env_uri=session_env1['environmentUri'],
+            tags=[session_id],
+        )
+        creds = generate_dataset_access_token(client1, ds.datasetUri)
+        dataset_session = boto3.Session(
+            aws_access_key_id=creds['AccessKey'],
+            aws_secret_access_key=creds['SessionKey'],
+            aws_session_token=creds['sessionToken'],
+        )
+        GlueClient(dataset_session, ds.region).create_table(
+            database_name=ds.GlueDatabaseName, table_name='integrationtest', bucket=ds.S3Bucket
+        )
+        response = sync_tables(client1, datasetUri=ds.datasetUri)
+
+        yield ds, response.get('nodes', [])[0]
+    finally:
+        if ds:
+            delete_s3_dataset(client1, session_env1['environmentUri'], ds)
+
+
 @pytest.fixture(scope='session')
 def session_imported_sse_s3_dataset1(
     client1, group1, org1, session_env1, session_id, testdata, session_env1_aws_client, resources_prefix

tests_new/integration_tests/modules/s3_datasets/test_s3_dataset.py

@@ -214,3 +214,116 @@ def test_get_dataset_presigned_url_upload_data_unauthorized(client2, session_s3_
     assert_that(get_dataset_presigned_role_url).raises(GqlError).when_called_with(
         client2, dataset_uri, input={'prefix': 'sample_data', 'fileName': 'name'}
     ).contains('UnauthorizedOperation', 'CREDENTIALS_DATASET', dataset_uri)
+
+
+
+
+def test_start_crawler(client1, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    response = start_glue_crawler(client1, datasetUri=dataset_uri, input=None)
+    assert_that(response.get('Name')).is_equal_to(session_s3_dataset1.GlueCrawlerName)
+    assert_that(response.get('status')).is_in(['Pending', 'Running'])
+    # TODO: check it can run successfully + check sending prefix
+
+
+def test_start_crawler_unauthorized(client2, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    assert_that(start_glue_crawler).raises(GqlError).when_called_with(client2, dataset_uri).contains(
+        'UnauthorizedOperation', 'CRAWL_DATASET', dataset_uri
+    )
+
+
+def test_sync_tables(client1, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    response = sync_tables(client1, datasetUri=dataset_uri)
+    assert_that(response.count).is_equal_to(2)
+
+
+def test_sync_tables_unauthorized(client2, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    assert_that(sync_tables).raises(GqlError).when_called_with(client2, dataset_uri).contains(
+        'UnauthorizedOperation', 'SYNC_DATASET', dataset_uri
+    )
+
+
+def test_start_table_profiling(client1, session_s3_dataset2_with_table):
+    dataset, table = session_s3_dataset2_with_table
+    table_uri = table.tableUri
+    dataset_uri = dataset.datasetUri
+    response = start_dataset_profiling_run(
+        client1, input={'datasetUri': dataset_uri, 'tableUri': table_uri, 'GlueTableName': table.GlueTableName}
+    )
+    assert_that(response.datasetUri).is_equal_to(dataset_uri)
+    assert_that(response.status).is_equal_to('RUNNING')
+    assert_that(response.GlueTableName).is_equal_to(table.GlueTableName)
+
+
+def test_start_table_profiling_unauthorized(client2, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    assert_that(start_dataset_profiling_run).raises(GqlError).when_called_with(client2, dataset_uri).contains(
+        'UnauthorizedOperation', 'PROFILE_DATASET_TABLE', dataset_uri
+    )
+
+
+def test_preview_table(client1, session_s3_dataset2_with_table):
+    dataset, table = session_s3_dataset2_with_table
+    table_uri = table.tableUri
+    response = preview_table(client1, table_uri)
+    assert_that(response.rows).exists()
+
+
+def test_preview_table_unauthorized(client2, session_s3_dataset2_with_table):
+    dataset, table = session_s3_dataset2_with_table
+    table_uri = table.tableUri
+    # TODO: confidentiality levels
+    assert_that(preview_table).raises(GqlError).when_called_with(client2, table_uri, {}).contains(
+        'UnauthorizedOperation', 'PREVIEW_DATASET_TABLE', table_uri
+    )
+
+
+def test_delete_table(client1, session_s3_dataset2_with_table):
+    dataset, table = session_s3_dataset2_with_table
+    # todo
+
+
+def test_delete_table_unauthorized(client2, session_s3_dataset2_with_table):
+    dataset, table = session_s3_dataset2_with_table
+    table_uri = table.tableUri
+    assert_that(delete_table).raises(GqlError).when_called_with(client2, table_uri).contains(
+        'UnauthorizedOperation', 'DELETE_DATASET_TABLE', table_uri
+    )
+
+
+def test_create_folder(client1, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    response = create_folder(
+        client1, datasetUri=dataset_uri, input={'prefix': 'folderCreatedInTest', 'label': 'labelFolder'}
+    )
+    assert_that(response.S3Prefix).is_equal_to('folderCreatedInTest')
+    assert_that(response.label).is_equal_to('labelFolder')
+
+
+def test_create_folder_unauthorized(client2, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    assert_that(create_folder).raises(GqlError).when_called_with(client2, dataset_uri, {}).contains(
+        'UnauthorizedOperation', 'CREATE_DATASET_FOLDER', dataset_uri
+    )
+
+
+def test_delete_folder(client1, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    location = create_folder(
+        client1, datasetUri=dataset_uri, input={'prefix': 'folderToDelete', 'label': 'folderToDelete'}
+    )
+    response = delete_folder(client1, location.locationUri)
+    assert_that(response).is_equal_to(True)
+
+
+def test_delete_folder_unauthorized(client1, client2, session_s3_dataset1):
+    dataset_uri = session_s3_dataset1.datasetUri
+    location = create_folder(
+        client1, datasetUri=dataset_uri, input={'prefix': 'folderToDelete', 'label': 'folderToDelete'}
+    )
+    assert_that(delete_folder).raises(GqlError).when_called_with(client2, location.locationUri).contains(
+        'UnauthorizedOperation', 'DELETE_DATASET_FOLDER', location.locationUri
+    )