-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
64 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
...erg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/TableIdGenerator.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
package io.airbyte.integrations.destination.iceberg.v2 | ||
|
||
import io.airbyte.cdk.load.command.DestinationStream | ||
import io.airbyte.cdk.load.command.iceberg.parquet.GlueCatalogConfiguration | ||
import io.micronaut.context.annotation.Factory | ||
import javax.inject.Singleton | ||
import org.apache.iceberg.catalog.Namespace | ||
import org.apache.iceberg.catalog.TableIdentifier | ||
|
||
/** | ||
* Convert our internal stream descriptor to an Iceberg [TableIdentifier]. | ||
* Implementations should handle catalog-specific naming restrictions. | ||
*/ | ||
// TODO accept default namespace in config as a val here | ||
interface TableIdGenerator { | ||
fun toTableIdentifier(stream: DestinationStream.Descriptor): TableIdentifier | ||
} | ||
|
||
class SimpleTableIdGenerator : TableIdGenerator { | ||
override fun toTableIdentifier(stream: DestinationStream.Descriptor): TableIdentifier = | ||
tableIdOf(stream.namespace!!, stream.name) | ||
} | ||
|
||
/** | ||
* AWS Glue requires lowercase database+table names. | ||
*/ | ||
class GlueTableIdGenerator : TableIdGenerator { | ||
override fun toTableIdentifier(stream: DestinationStream.Descriptor): TableIdentifier = | ||
tableIdOf(stream.namespace!!.lowercase(), stream.name.lowercase()) | ||
} | ||
|
||
@Factory | ||
class TableIdGeneratorFactory(private val icebergConfiguration: IcebergV2Configuration) { | ||
@Singleton | ||
fun create() = | ||
when (icebergConfiguration.icebergCatalogConfiguration.catalogConfiguration) { | ||
is GlueCatalogConfiguration -> GlueTableIdGenerator() | ||
else -> SimpleTableIdGenerator() | ||
} | ||
} | ||
|
||
// iceberg namespace+name must both be nonnull. | ||
private fun tableIdOf(namespace: String, name: String) = | ||
TableIdentifier.of(Namespace.of(namespace), name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters