Skip to content

Commit

Permalink
Change identificatie column type to char(4) (#1435)
Browse files Browse the repository at this point in the history 
* WIP add integration test for loading woonplaats stand and mutatie

* Rework, add Oracle support

* Change identificatie column type of Woonplaats to char(4) because Oracle requires queries with space padded values, fixes SUPPORT-13151

* Catch table does not exist error
  • Loading branch information
@matthijsln
matthijsln authored Jun 2, 2022
1 parent f962d86 commit 9de9939
Show file tree
Hide file tree
Showing 10 changed files with 503 additions and 10 deletions.
14 changes: 12 additions & 2 deletions bag2-loader/src/main/java/nl/b3p/brmo/bag2/schema/BAG2Schema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,20 @@ private List<AttributeColumnMapping> withBaseAttributes(AttributeColumnMapping..
public BAG2Schema() {
super();

addObjectType(new BAG2ObjectType(this, "Woonplaats", withBaseAttributes(
// For Woonplaats, use CHAR(4) instead of CHAR(16). For Oracle we can't use CHAR(16) because then we would need
// to query with space-padded values to correctly delete previous versions on mutaties (issue 13151).
List<AttributeColumnMapping> woonplaatsAttributes = bag2BaseAttributes.stream().map(attribute -> {
if ("identificatie".equals(attribute.getName())) {
return new AttributeColumnMapping("identificatie", "char(4)", true, true);
}
return attribute;
}).collect(Collectors.toList());
woonplaatsAttributes.addAll(List.of(
new AttributeColumnMapping("naam"),
new GeometryAttributeColumnMapping("geometrie", "geometry(GEOMETRY, 28992)")
)).addExtraDataDefinitionSQL(List.of(
));
addObjectType(new BAG2ObjectType(this, "Woonplaats", woonplaatsAttributes
).addExtraDataDefinitionSQL(List.of(
"create or replace view v_woonplaats_actueel as select * from woonplaats where " + WHERE_CLAUSE_ACTUEEL
)));

Expand Down
110 changes: 102 additions & 8 deletions bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/BAGLoaderDatabaseIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,35 @@
import nl.b3p.brmo.bag2.loader.cli.BAG2DatabaseOptions;
import nl.b3p.brmo.bag2.loader.cli.BAG2LoadOptions;
import nl.b3p.brmo.bag2.loader.cli.BAG2LoaderMain;
import nl.b3p.brmo.sql.LoggingQueryRunner;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dbunit.Assertion;
import org.dbunit.IDatabaseTester;
import org.dbunit.JdbcDatabaseTester;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.DefaultMetadataHandler;
import org.dbunit.database.IDatabaseConnection;
import org.dbunit.database.IMetadataHandler;
import org.dbunit.dataset.IDataSet;
import org.dbunit.dataset.xml.XmlDataSet;
import org.dbunit.ext.oracle.Oracle10DataTypeFactory;
import org.dbunit.ext.postgresql.PostgresqlDataTypeFactory;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;

import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;

import static org.junit.jupiter.api.Assertions.assertTrue;
Expand All @@ -35,7 +51,10 @@
*
* @author mprins
*/
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class BAGLoaderDatabaseIntegrationTest {
private static final Log LOG = LogFactory.getLog(BAGLoaderDatabaseIntegrationTest.class);

private static final String[] BAGTABLES = new String[]{"ligplaats", "ligplaats_nevenadres", "nummeraanduiding", "openbareruimte", "pand", "standplaats", "standplaats_nevenadres", "verblijfsobject", "verblijfsobject_gebruiksdoel", "verblijfsobject_maaktdeeluitvan", "verblijfsobject_nevenadres", "woonplaats"};
private static final String[] BAGACTUEELVIEWS = new String[]{"v_ligplaats_actueel", "v_nummeraanduiding_actueel", "v_openbareruimte_actueel", "v_pand_actueel", "v_standplaats_actueel", "v_verblijfsobject_actueel", "v_woonplaats_actueel"};
private static String dbUrl = System.getProperty("dburl");
Expand All @@ -48,14 +67,16 @@ public class BAGLoaderDatabaseIntegrationTest {
private IDatabaseConnection bag;
private String testFileName;
private String tableQualifierPrefix = "";
private String schema = null;
private String expectedXmlDataSetSuffix = "";

@BeforeAll
static void beforeAll() {
BAG2LoaderMain.configureLogging(false);
}

@BeforeEach
void setUp() throws Exception {
void setUp(TestInfo info) throws Exception {
assumeFalse(StringUtils.isEmpty(dbUrl), "skipping integration test: missing database url");
URL u = BAGLoaderDatabaseIntegrationTest.class.getResource("/BAGGEM1904L-15102021.zip");
assumeFalse(null == u, "skipping integration test: missing testdata");
Expand All @@ -66,10 +87,15 @@ void setUp() throws Exception {
bag.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new PostgresqlDataTypeFactory());
bag.getConfig().setProperty(DatabaseConfig.FEATURE_QUALIFIED_TABLE_NAMES, true);
tableQualifierPrefix = "bag.";
schema = "bag";
}
if (dbUrl.contains("oracle")) {
bag.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new Oracle10DataTypeFactory());
bag.getConfig().setProperty(DatabaseConfig.FEATURE_SKIP_ORACLE_RECYCLEBIN_TABLES, true);

expectedXmlDataSetSuffix = "-oracle";
// If we don't set the schema to the user the DatabaseMetaData.getTables() call without a schema filter will take 5 minutes
schema = dbUser.toUpperCase();
}

bag2LoadOptions = new BAG2LoadOptions();
Expand All @@ -82,23 +108,91 @@ void setUp() throws Exception {
databaseOptions.setUser(dbUser);
databaseOptions.setPassword(dbPass);
bag2Database = new BAG2Database(databaseOptions);

if (info.getTestMethod().isPresent()) {
if(info.getTestMethod().get().getAnnotation(SkipDropTables.class) == null) {
try {
dropTables(bag.getConnection(), schema, dbUrl.contains("oracle"));
} catch(Exception e) {
LOG.error("Exception dropping tables before test", e);
}
}
}
}

private static void dropTables(Connection connection, String schema, boolean isOracle) throws SQLException {
if (!isOracle) {
LOG.trace("Drop BAG schema");
new LoggingQueryRunner().update(connection,"drop schema if exists " + schema + " cascade");
} else {
IMetadataHandler metadataHandler = new DefaultMetadataHandler();
try (ResultSet tablesRs = metadataHandler.getTables(connection.getMetaData(), schema, new String[] { "TABLE" })) {
while(tablesRs.next()) {
String tableName = tablesRs.getString("TABLE_NAME");
try {
LOG.trace("Drop table: " + tableName);
new LoggingQueryRunner().update(connection, "drop table " + tableName + " cascade constraints");
} catch (SQLException se) {
LOG.warn("Exception dropping table " + tableName + ": " + se.getLocalizedMessage());
}
}
}
new LoggingQueryRunner().update(connection,"delete from user_sdo_geom_metadata");
}
}

@AfterEach
void cleanup() throws SQLException {
// geladen data wordt niet opgeruimd
if (null != bag) bag.close();
}

@Test
void testStand() throws SQLException {
BAG2LoaderMain loader = new BAG2LoaderMain();

private void loadBAGResourceFile(String file) {
try {
loader.loadFiles(bag2Database, databaseOptions, bag2LoadOptions, new BAG2ProgressReporter(), new String[]{testFileName}, null);
BAG2LoaderMain loader = new BAG2LoaderMain();
loader.loadFiles(bag2Database, databaseOptions, bag2LoadOptions, new BAG2ProgressReporter(), new String[]{ file }, null);
} catch (Exception e) {
fail("Laden BAG data is mislukt. " + e.getLocalizedMessage(), e);
fail("Laden BAG data uit resource " + file + " is mislukt: " + e.getLocalizedMessage(), e);
}
}

private void compareDataSet(String[] tables, String expectedXmlDataSetFileName) throws Exception {
IDatabaseTester databaseTester = new JdbcDatabaseTester(bag2Database.getDialect().getDriverClass(), databaseOptions.getConnectionString(), databaseOptions.getUser(), databaseOptions.getPassword(), schema);
IDatabaseConnection dbTestConnection = databaseTester.getConnection();
IDataSet actualDataSet = dbTestConnection.createDataSet(tables);
if (System.getProperty("db.writeActualDataSet") != null) {
XmlDataSet.write(actualDataSet, System.out);
}
IDataSet expectedDataSet = new XmlDataSet(BAGLoaderDatabaseIntegrationTest.class.getResource(String.format("/expected/%s%s.xml", expectedXmlDataSetFileName, expectedXmlDataSetSuffix)).openStream());
for (String table: tables) {
Assertion.assertEqualsIgnoreCols(expectedDataSet, actualDataSet, table, new String[] { "objectid" });
}
}

private static String getResourceFile(String resource) {
return BAGLoaderDatabaseIntegrationTest.class.getResource(resource).getFile();
}

@Test
@Order(1)
void testCompareDataSetWoonplaatsStand() throws Exception {
loadBAGResourceFile(getResourceFile("/BAGGEM3502L-15102021.zip"));
compareDataSet(new String[] { "woonplaats"}, "bag2-woonplaats-stand");
}

@Test
@Order(2)
@SkipDropTables
void testCompareDataSetWoonplaatsMutatie() throws Exception {
loadBAGResourceFile(getResourceFile("/BAGNLDM-23052022-24052022.zip"));
compareDataSet(new String[] { "woonplaats"}, "bag2-woonplaats-gemuteerd");
}

// Leave this as last integration test case, so the BAG tables loaded by it can be used to test the create view scripts
@Test
@Order(Integer.MAX_VALUE)
void testStandAllTablesAndViewsHaveRows() throws Exception {
loadBAGResourceFile(testFileName);

// check tables
for (String t : BAGTABLES) {
// omdat sommige BAG tabellen ook in RSGB schema zitten bag qualifier gebruiken
Expand Down
18 changes: 18 additions & 0 deletions bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/SkipDropTables.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyright (C) 2021 B3Partners B.V.
*
* SPDX-License-Identifier: MIT
*
*/

package nl.b3p.brmo.bag2.loader;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ ElementType.ANNOTATION_TYPE, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
public @interface SkipDropTables {
}
3 changes: 3 additions & 0 deletions bag2-loader/src/test/resources/BAGGEM3502L-15102021.zip
View file
Git LFS file not shown
3 changes: 3 additions & 0 deletions bag2-loader/src/test/resources/BAGNLDM-23052022-24052022.zip
View file
Git LFS file not shown
93 changes: 93 additions & 0 deletions bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd-oracle.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
<?xml version='1.0' encoding='UTF-8'?>
<dataset>
<table name="WOONPLAATS">
<column>OBJECTID</column>
<column>IDENTIFICATIE</column>
<column>VOORKOMENIDENTIFICATIE</column>
<column>BEGINGELDIGHEID</column>
<column>EINDGELDIGHEID</column>
<column>TIJDSTIPREGISTRATIE</column>
<column>EINDREGISTRATIE</column>
<column>TIJDSTIPINACTIEF</column>
<column>TIJDSTIPREGISTRATIELV</column>
<column>TIJDSTIPEINDREGISTRATIELV</column>
<column>TIJDSTIPINACTIEFLV</column>
<column>DOCUMENTDATUM</column>
<column>DOCUMENTNUMMER</column>
<column>GECONSTATEERD</column>
<column>STATUS</column>
<column>NAAM</column>
<row>
<value>1</value>
<value>3502</value>
<value>1</value>
<value><![CDATA[2010-09-21 00:00:00.0]]></value>
<value><![CDATA[2011-09-01 00:00:00.0]]></value>
<value><![CDATA[2010-12-31 11:25:43.0]]></value>
<value><![CDATA[2011-09-08 12:20:45.0]]></value>
<null/>
<value><![CDATA[2010-12-31 11:33:38.0]]></value>
<value><![CDATA[2011-09-08 12:31:38.0]]></value>
<null/>
<value><![CDATA[2010-09-21 00:00:00.0]]></value>
<value>AVGpvdb02</value>
<value>0</value>
<value><![CDATA[Woonplaats aangewezen]]></value>
<value>NIEUWKUIJK</value>
</row>
<row>
<value>2</value>
<value>3502</value>
<value>2</value>
<value><![CDATA[2011-09-01 00:00:00.0]]></value>
<value><![CDATA[2011-10-05 00:00:00.0]]></value>
<value><![CDATA[2011-09-08 12:20:45.0]]></value>
<value><![CDATA[2011-10-05 12:47:19.0]]></value>
<null/>
<value><![CDATA[2011-09-08 12:31:38.0]]></value>
<value><![CDATA[2011-10-05 13:01:44.0]]></value>
<null/>
<value><![CDATA[2011-09-01 00:00:00.0]]></value>
<value>AVGEL01092011-04</value>
<value>0</value>
<value><![CDATA[Woonplaats aangewezen]]></value>
<value>Nieuwkuijk</value>
</row>
<row>
<value>4</value>
<value>3502</value>
<value>3</value>
<value><![CDATA[2011-10-05 00:00:00.0]]></value>
<value><![CDATA[2022-05-23 00:00:00.0]]></value>
<value><![CDATA[2011-10-05 12:47:19.0]]></value>
<value><![CDATA[2022-05-23 09:53:06.0]]></value>
<null/>
<value><![CDATA[2011-10-05 13:01:44.0]]></value>
<value><![CDATA[2022-05-23 09:53:21.0]]></value>
<null/>
<value><![CDATA[2011-10-05 00:00:00.0]]></value>
<value>AVGSK05102011-01</value>
<value>0</value>
<value><![CDATA[Woonplaats aangewezen]]></value>
<value>Nieuwkuijk</value>
</row>
<row>
<value>5</value>
<value>3502</value>
<value>4</value>
<value><![CDATA[2022-05-23 00:00:00.0]]></value>
<null/>
<value><![CDATA[2022-05-23 09:53:06.0]]></value>
<null/>
<null/>
<value><![CDATA[2022-05-23 09:53:21.0]]></value>
<null/>
<null/>
<value><![CDATA[2022-05-23 00:00:00.0]]></value>
<value>2068944</value>
<value>0</value>
<value><![CDATA[Woonplaats aangewezen]]></value>
<value>Nieuwkuijk</value>
</row>
</table>
</dataset>
Loading

0 comments on commit 9de9939

Please sign in to comment.