Skip to content

Commit

Permalink
MTT on S3 w/ CAS
Browse files Browse the repository at this point in the history
  • Loading branch information
cdouglas committed Dec 2, 2024
1 parent 74a5454 commit 5bc3a52
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,11 @@
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestWatcher;
import software.amazon.awssdk.services.s3.S3Client;

@ExtendWith(TestS3Catalog.SuccessCleanupExtension.class)
public class TestS3Catalog extends CatalogTests<FileIOCatalog> {
private static final String TEST_BUCKET = "casalog";

private static S3Client s3;
private static String uniqTestRun;
private static String warehouseLocation;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
*
* * Licensed to the Apache Software Foundation (ASF) under one
* * or more contributor license agreements. See the NOTICE file
* * distributed with this work for additional information
* * regarding copyright ownership. The ASF licenses this file
* * to you under the Apache License, Version 2.0 (the
* * "License"); you may not use this file except in compliance
* * with the License. You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing,
* * software distributed under the License is distributed on an
* * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* * KIND, either express or implied. See the License for the
* * specific language governing permissions and limitations
* * under the License.
*
*/

package org.apache.iceberg.aws.s3;

import org.apache.iceberg.catalog.CatalogTransactionTests;
import org.apache.iceberg.io.FileIOCatalog;
import java.util.Map;
import java.util.UUID;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestWatcher;

@ExtendWith(TestS3Catalog.SuccessCleanupExtension.class)
public class TestS3FileIOCatalogTransaction extends CatalogTransactionTests<FileIOCatalog> {
private static final String TEST_BUCKET = "casalog";

private static String uniqTestRun;
private static String warehouseLocation;

private FileIOCatalog catalog;

// Don't keep artifacts from successful tests
static class SuccessCleanupExtension implements TestWatcher {
@Override
public void testSuccessful(ExtensionContext ctxt) {
cleanupWarehouseLocation();
}
}

static void cleanupWarehouseLocation() {
// TODO use FileIO
}

@BeforeAll
public static void initStorage() {
uniqTestRun = UUID.randomUUID().toString();
// LOG.info("TEST RUN: {}", uniqTestRun);
System.err.println("TEST RUN: " + uniqTestRun); // (logging disabled in tests)
}

@BeforeEach
public void before(TestInfo info) {
final String testName = info.getTestMethod().orElseThrow(RuntimeException::new).getName();
warehouseLocation =
"s3://" + TEST_BUCKET + "/" + uniqTestRun + "/" + testName + "_" + info.getDisplayName();
cleanupWarehouseLocation();

final S3FileIO io = new S3FileIO(); // () -> s3);
io.initialize(Maps.newHashMap());
final String location = warehouseLocation + "/catalog";
catalog = new FileIOCatalog("test", location, null, io, Maps.newHashMap());

final Map<String, String> properties = Maps.newHashMap();
properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation);
catalog.initialize(testName, properties);
}

@AfterEach
public void after() {
// TODO
}

@Override
protected FileIOCatalog catalog() {
return catalog;
}
}
2 changes: 1 addition & 1 deletion vldb/schedule.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ tl;dr

## Aggressive Schedule

Week 1: CAS Catalog (S3) \
Week 1: CAS Catalog (S3) \
Week 2: Embed Table Metadata \
Week 3: ibid \
Week 4: Split table entries \
Expand Down

0 comments on commit 5bc3a52

Please sign in to comment.