From 9de99390d0a527d12ab4bb7572ac756d47792786 Mon Sep 17 00:00:00 2001 From: Matthijs Date: Thu, 2 Jun 2022 17:02:51 +0200 Subject: [PATCH] Change identificatie column type to char(4) (#1435) * WIP add integration test for loading woonplaats stand and mutatie * Rework, add Oracle support * Change identificatie column type of Woonplaats to char(4) because Oracle requires queries with space padded values, fixes SUPPORT-13151 * Catch table does not exist error --- .../nl/b3p/brmo/bag2/schema/BAG2Schema.java | 14 ++- .../BAGLoaderDatabaseIntegrationTest.java | 110 ++++++++++++++++-- .../b3p/brmo/bag2/loader/SkipDropTables.java | 18 +++ .../test/resources/BAGGEM3502L-15102021.zip | 3 + .../resources/BAGNLDM-23052022-24052022.zip | 3 + .../bag2-woonplaats-gemuteerd-oracle.xml | 93 +++++++++++++++ .../expected/bag2-woonplaats-gemuteerd.xml | 100 ++++++++++++++++ .../expected/bag2-woonplaats-stand-oracle.xml | 75 ++++++++++++ .../expected/bag2-woonplaats-stand.xml | 82 +++++++++++++ .../2.2.2-2.3.0/oracle/bag.sql | 15 +++ 10 files changed, 503 insertions(+), 10 deletions(-) create mode 100644 bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/SkipDropTables.java create mode 100644 bag2-loader/src/test/resources/BAGGEM3502L-15102021.zip create mode 100644 bag2-loader/src/test/resources/BAGNLDM-23052022-24052022.zip create mode 100644 bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd-oracle.xml create mode 100644 bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd.xml create mode 100644 bag2-loader/src/test/resources/expected/bag2-woonplaats-stand-oracle.xml create mode 100644 bag2-loader/src/test/resources/expected/bag2-woonplaats-stand.xml diff --git a/bag2-loader/src/main/java/nl/b3p/brmo/bag2/schema/BAG2Schema.java b/bag2-loader/src/main/java/nl/b3p/brmo/bag2/schema/BAG2Schema.java index ecd9beb239..4915958da8 100644 --- a/bag2-loader/src/main/java/nl/b3p/brmo/bag2/schema/BAG2Schema.java +++ b/bag2-loader/src/main/java/nl/b3p/brmo/bag2/schema/BAG2Schema.java @@ -77,10 +77,20 @@ private List withBaseAttributes(AttributeColumnMapping.. public BAG2Schema() { super(); - addObjectType(new BAG2ObjectType(this, "Woonplaats", withBaseAttributes( + // For Woonplaats, use CHAR(4) instead of CHAR(16). For Oracle we can't use CHAR(16) because then we would need + // to query with space-padded values to correctly delete previous versions on mutaties (issue 13151). + List woonplaatsAttributes = bag2BaseAttributes.stream().map(attribute -> { + if ("identificatie".equals(attribute.getName())) { + return new AttributeColumnMapping("identificatie", "char(4)", true, true); + } + return attribute; + }).collect(Collectors.toList()); + woonplaatsAttributes.addAll(List.of( new AttributeColumnMapping("naam"), new GeometryAttributeColumnMapping("geometrie", "geometry(GEOMETRY, 28992)") - )).addExtraDataDefinitionSQL(List.of( + )); + addObjectType(new BAG2ObjectType(this, "Woonplaats", woonplaatsAttributes + ).addExtraDataDefinitionSQL(List.of( "create or replace view v_woonplaats_actueel as select * from woonplaats where " + WHERE_CLAUSE_ACTUEEL ))); diff --git a/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/BAGLoaderDatabaseIntegrationTest.java b/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/BAGLoaderDatabaseIntegrationTest.java index 1c5c552425..76632cbcb8 100644 --- a/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/BAGLoaderDatabaseIntegrationTest.java +++ b/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/BAGLoaderDatabaseIntegrationTest.java @@ -9,19 +9,35 @@ import nl.b3p.brmo.bag2.loader.cli.BAG2DatabaseOptions; import nl.b3p.brmo.bag2.loader.cli.BAG2LoadOptions; import nl.b3p.brmo.bag2.loader.cli.BAG2LoaderMain; +import nl.b3p.brmo.sql.LoggingQueryRunner; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.dbunit.Assertion; +import org.dbunit.IDatabaseTester; +import org.dbunit.JdbcDatabaseTester; import org.dbunit.database.DatabaseConfig; import org.dbunit.database.DatabaseConnection; +import org.dbunit.database.DefaultMetadataHandler; import org.dbunit.database.IDatabaseConnection; +import org.dbunit.database.IMetadataHandler; +import org.dbunit.dataset.IDataSet; +import org.dbunit.dataset.xml.XmlDataSet; import org.dbunit.ext.oracle.Oracle10DataTypeFactory; import org.dbunit.ext.postgresql.PostgresqlDataTypeFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; import java.net.URL; +import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,7 +51,10 @@ * * @author mprins */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class BAGLoaderDatabaseIntegrationTest { + private static final Log LOG = LogFactory.getLog(BAGLoaderDatabaseIntegrationTest.class); + private static final String[] BAGTABLES = new String[]{"ligplaats", "ligplaats_nevenadres", "nummeraanduiding", "openbareruimte", "pand", "standplaats", "standplaats_nevenadres", "verblijfsobject", "verblijfsobject_gebruiksdoel", "verblijfsobject_maaktdeeluitvan", "verblijfsobject_nevenadres", "woonplaats"}; private static final String[] BAGACTUEELVIEWS = new String[]{"v_ligplaats_actueel", "v_nummeraanduiding_actueel", "v_openbareruimte_actueel", "v_pand_actueel", "v_standplaats_actueel", "v_verblijfsobject_actueel", "v_woonplaats_actueel"}; private static String dbUrl = System.getProperty("dburl"); @@ -48,6 +67,8 @@ public class BAGLoaderDatabaseIntegrationTest { private IDatabaseConnection bag; private String testFileName; private String tableQualifierPrefix = ""; + private String schema = null; + private String expectedXmlDataSetSuffix = ""; @BeforeAll static void beforeAll() { @@ -55,7 +76,7 @@ static void beforeAll() { } @BeforeEach - void setUp() throws Exception { + void setUp(TestInfo info) throws Exception { assumeFalse(StringUtils.isEmpty(dbUrl), "skipping integration test: missing database url"); URL u = BAGLoaderDatabaseIntegrationTest.class.getResource("/BAGGEM1904L-15102021.zip"); assumeFalse(null == u, "skipping integration test: missing testdata"); @@ -66,10 +87,15 @@ void setUp() throws Exception { bag.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new PostgresqlDataTypeFactory()); bag.getConfig().setProperty(DatabaseConfig.FEATURE_QUALIFIED_TABLE_NAMES, true); tableQualifierPrefix = "bag."; + schema = "bag"; } if (dbUrl.contains("oracle")) { bag.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new Oracle10DataTypeFactory()); bag.getConfig().setProperty(DatabaseConfig.FEATURE_SKIP_ORACLE_RECYCLEBIN_TABLES, true); + + expectedXmlDataSetSuffix = "-oracle"; + // If we don't set the schema to the user the DatabaseMetaData.getTables() call without a schema filter will take 5 minutes + schema = dbUser.toUpperCase(); } bag2LoadOptions = new BAG2LoadOptions(); @@ -82,23 +108,91 @@ void setUp() throws Exception { databaseOptions.setUser(dbUser); databaseOptions.setPassword(dbPass); bag2Database = new BAG2Database(databaseOptions); + + if (info.getTestMethod().isPresent()) { + if(info.getTestMethod().get().getAnnotation(SkipDropTables.class) == null) { + try { + dropTables(bag.getConnection(), schema, dbUrl.contains("oracle")); + } catch(Exception e) { + LOG.error("Exception dropping tables before test", e); + } + } + } + } + + private static void dropTables(Connection connection, String schema, boolean isOracle) throws SQLException { + if (!isOracle) { + LOG.trace("Drop BAG schema"); + new LoggingQueryRunner().update(connection,"drop schema if exists " + schema + " cascade"); + } else { + IMetadataHandler metadataHandler = new DefaultMetadataHandler(); + try (ResultSet tablesRs = metadataHandler.getTables(connection.getMetaData(), schema, new String[] { "TABLE" })) { + while(tablesRs.next()) { + String tableName = tablesRs.getString("TABLE_NAME"); + try { + LOG.trace("Drop table: " + tableName); + new LoggingQueryRunner().update(connection, "drop table " + tableName + " cascade constraints"); + } catch (SQLException se) { + LOG.warn("Exception dropping table " + tableName + ": " + se.getLocalizedMessage()); + } + } + } + new LoggingQueryRunner().update(connection,"delete from user_sdo_geom_metadata"); + } } @AfterEach void cleanup() throws SQLException { - // geladen data wordt niet opgeruimd if (null != bag) bag.close(); } - @Test - void testStand() throws SQLException { - BAG2LoaderMain loader = new BAG2LoaderMain(); - + private void loadBAGResourceFile(String file) { try { - loader.loadFiles(bag2Database, databaseOptions, bag2LoadOptions, new BAG2ProgressReporter(), new String[]{testFileName}, null); + BAG2LoaderMain loader = new BAG2LoaderMain(); + loader.loadFiles(bag2Database, databaseOptions, bag2LoadOptions, new BAG2ProgressReporter(), new String[]{ file }, null); } catch (Exception e) { - fail("Laden BAG data is mislukt. " + e.getLocalizedMessage(), e); + fail("Laden BAG data uit resource " + file + " is mislukt: " + e.getLocalizedMessage(), e); + } + } + + private void compareDataSet(String[] tables, String expectedXmlDataSetFileName) throws Exception { + IDatabaseTester databaseTester = new JdbcDatabaseTester(bag2Database.getDialect().getDriverClass(), databaseOptions.getConnectionString(), databaseOptions.getUser(), databaseOptions.getPassword(), schema); + IDatabaseConnection dbTestConnection = databaseTester.getConnection(); + IDataSet actualDataSet = dbTestConnection.createDataSet(tables); + if (System.getProperty("db.writeActualDataSet") != null) { + XmlDataSet.write(actualDataSet, System.out); + } + IDataSet expectedDataSet = new XmlDataSet(BAGLoaderDatabaseIntegrationTest.class.getResource(String.format("/expected/%s%s.xml", expectedXmlDataSetFileName, expectedXmlDataSetSuffix)).openStream()); + for (String table: tables) { + Assertion.assertEqualsIgnoreCols(expectedDataSet, actualDataSet, table, new String[] { "objectid" }); } + } + + private static String getResourceFile(String resource) { + return BAGLoaderDatabaseIntegrationTest.class.getResource(resource).getFile(); + } + + @Test + @Order(1) + void testCompareDataSetWoonplaatsStand() throws Exception { + loadBAGResourceFile(getResourceFile("/BAGGEM3502L-15102021.zip")); + compareDataSet(new String[] { "woonplaats"}, "bag2-woonplaats-stand"); + } + + @Test + @Order(2) + @SkipDropTables + void testCompareDataSetWoonplaatsMutatie() throws Exception { + loadBAGResourceFile(getResourceFile("/BAGNLDM-23052022-24052022.zip")); + compareDataSet(new String[] { "woonplaats"}, "bag2-woonplaats-gemuteerd"); + } + + // Leave this as last integration test case, so the BAG tables loaded by it can be used to test the create view scripts + @Test + @Order(Integer.MAX_VALUE) + void testStandAllTablesAndViewsHaveRows() throws Exception { + loadBAGResourceFile(testFileName); + // check tables for (String t : BAGTABLES) { // omdat sommige BAG tabellen ook in RSGB schema zitten bag qualifier gebruiken diff --git a/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/SkipDropTables.java b/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/SkipDropTables.java new file mode 100644 index 0000000000..fda8a067eb --- /dev/null +++ b/bag2-loader/src/test/java/nl/b3p/brmo/bag2/loader/SkipDropTables.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2021 B3Partners B.V. + * + * SPDX-License-Identifier: MIT + * + */ + +package nl.b3p.brmo.bag2.loader; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ ElementType.ANNOTATION_TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +public @interface SkipDropTables { +} diff --git a/bag2-loader/src/test/resources/BAGGEM3502L-15102021.zip b/bag2-loader/src/test/resources/BAGGEM3502L-15102021.zip new file mode 100644 index 0000000000..dc3c1b8b8b --- /dev/null +++ b/bag2-loader/src/test/resources/BAGGEM3502L-15102021.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605a598e5ef6ac8330c475e31ccf45ebb978459cea01e9d934017b0ce885d5f3 +size 13544 diff --git a/bag2-loader/src/test/resources/BAGNLDM-23052022-24052022.zip b/bag2-loader/src/test/resources/BAGNLDM-23052022-24052022.zip new file mode 100644 index 0000000000..f249f8db56 --- /dev/null +++ b/bag2-loader/src/test/resources/BAGNLDM-23052022-24052022.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d99bd5e3cb06372ae75e68629727a2578360edca3bd0fdfd812bf851cfa16f +size 6947 diff --git a/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd-oracle.xml b/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd-oracle.xml new file mode 100644 index 0000000000..be9db18f81 --- /dev/null +++ b/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd-oracle.xml @@ -0,0 +1,93 @@ + + + + OBJECTID + IDENTIFICATIE + VOORKOMENIDENTIFICATIE + BEGINGELDIGHEID + EINDGELDIGHEID + TIJDSTIPREGISTRATIE + EINDREGISTRATIE + TIJDSTIPINACTIEF + TIJDSTIPREGISTRATIELV + TIJDSTIPEINDREGISTRATIELV + TIJDSTIPINACTIEFLV + DOCUMENTDATUM + DOCUMENTNUMMER + GECONSTATEERD + STATUS + NAAM + + 1 + 3502 + 1 + + + + + + + + + + AVGpvdb02 + 0 + + NIEUWKUIJK + + + 2 + 3502 + 2 + + + + + + + + + + AVGEL01092011-04 + 0 + + Nieuwkuijk + + + 4 + 3502 + 3 + + + + + + + + + + AVGSK05102011-01 + 0 + + Nieuwkuijk + + + 5 + 3502 + 4 + + + + + + + + + + 2068944 + 0 + + Nieuwkuijk + +
+ \ No newline at end of file diff --git a/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd.xml b/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd.xml new file mode 100644 index 0000000000..7b01d3c25b --- /dev/null +++ b/bag2-loader/src/test/resources/expected/bag2-woonplaats-gemuteerd.xml @@ -0,0 +1,100 @@ + + + + + + objectid + identificatie + voorkomenidentificatie + begingeldigheid + eindgeldigheid + tijdstipregistratie + eindregistratie + tijdstipinactief + tijdstipregistratielv + tijdstipeindregistratielv + tijdstipinactieflv + documentdatum + documentnummer + geconstateerd + status + naam + + 1 + 3502 + 1 + 2010-09-21 + 2011-09-01 + + + + + + + 2010-09-21 + AVGpvdb02 + false + + NIEUWKUIJK + + + 2 + 3502 + 2 + 2011-09-01 + 2011-10-05 + + + + + + + 2011-09-01 + AVGEL01092011-04 + false + + Nieuwkuijk + + + 4 + 3502 + 3 + 2011-10-05 + 2022-05-23 + + + + + + + 2011-10-05 + AVGSK05102011-01 + false + + Nieuwkuijk + + + 5 + 3502 + 4 + 2022-05-23 + + + + + + + + 2022-05-23 + 2068944 + false + + Nieuwkuijk + +
+ \ No newline at end of file diff --git a/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand-oracle.xml b/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand-oracle.xml new file mode 100644 index 0000000000..55ddf1bea3 --- /dev/null +++ b/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand-oracle.xml @@ -0,0 +1,75 @@ + + + + OBJECTID + IDENTIFICATIE + VOORKOMENIDENTIFICATIE + BEGINGELDIGHEID + EINDGELDIGHEID + TIJDSTIPREGISTRATIE + EINDREGISTRATIE + TIJDSTIPINACTIEF + TIJDSTIPREGISTRATIELV + TIJDSTIPEINDREGISTRATIELV + TIJDSTIPINACTIEFLV + DOCUMENTDATUM + DOCUMENTNUMMER + GECONSTATEERD + STATUS + NAAM + + 1 + 3502 + 1 + + + + + + + + + + AVGpvdb02 + 0 + + NIEUWKUIJK + + + 2 + 3502 + 2 + + + + + + + + + + AVGEL01092011-04 + 0 + + Nieuwkuijk + + + 3 + 3502 + 3 + + + + + + + + + + AVGSK05102011-01 + 0 + + Nieuwkuijk + +
+ \ No newline at end of file diff --git a/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand.xml b/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand.xml new file mode 100644 index 0000000000..0b1c3676fd --- /dev/null +++ b/bag2-loader/src/test/resources/expected/bag2-woonplaats-stand.xml @@ -0,0 +1,82 @@ + + + + + + objectid + identificatie + voorkomenidentificatie + begingeldigheid + eindgeldigheid + tijdstipregistratie + eindregistratie + tijdstipinactief + tijdstipregistratielv + tijdstipeindregistratielv + tijdstipinactieflv + documentdatum + documentnummer + geconstateerd + status + naam + + 1 + 3502 + 1 + 2010-09-21 + 2011-09-01 + + + + + + + 2010-09-21 + AVGpvdb02 + false + + NIEUWKUIJK + + + 2 + 3502 + 2 + 2011-09-01 + 2011-10-05 + + + + + + + 2011-09-01 + AVGEL01092011-04 + false + + Nieuwkuijk + + + 3 + 3502 + 3 + 2011-10-05 + + + + + + + + 2011-10-05 + AVGSK05102011-01 + false + + Nieuwkuijk + +
+ \ No newline at end of file diff --git a/datamodel/upgrade_scripts/2.2.2-2.3.0/oracle/bag.sql b/datamodel/upgrade_scripts/2.2.2-2.3.0/oracle/bag.sql index 12043fa69c..b54cec24c7 100644 --- a/datamodel/upgrade_scripts/2.2.2-2.3.0/oracle/bag.sql +++ b/datamodel/upgrade_scripts/2.2.2-2.3.0/oracle/bag.sql @@ -16,6 +16,21 @@ END; / MERGE INTO brmo_metadata USING DUAL ON (naam = 'brmoversie') WHEN NOT MATCHED THEN INSERT (naam) VALUES('brmoversie'); +-- Fix voor SUPPORT-13151 +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN +EXECUTE IMMEDIATE 'ALTER TABLE WOONPLAATS MODIFY IDENTIFICATIE VARCHAR(16)'; +EXECUTE IMMEDIATE 'UPDATE WOONPLAATS SET IDENTIFICATIE = TRIM(IDENTIFICATIE)'; +EXECUTE IMMEDIATE 'ALTER TABLE WOONPLAATS MODIFY IDENTIFICATIE CHAR(4)'; +EXCEPTION +WHEN OTHERS THEN +IF + SQLCODE = -942 THEN + NULL; +ELSE RAISE; +END IF; +END; +/ -- onderstaande dienen als laatste stappen van een upgrade uitgevoerd INSERT INTO brmo_metadata (naam,waarde) SELECT 'upgrade_2.2.2_naar_2.3.0','vorige versie was ' || waarde FROM brmo_metadata WHERE naam='brmoversie';