Add support for hidden or invisible columns in MySQL and MariaDB

schemacrawler-api/src/main/java/schemacrawler/crawl/TableColumnRetriever.java

@@ -38,9 +38,7 @@
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.Arrays;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
@@ -82,7 +80,7 @@ void retrieveTableColumns(
       return;
     }
 
-    final Set<List<String>> hiddenTableColumnsLookupKeys = retrieveHiddenTableColumnsLookupKeys();
+    final Set<NamedObjectKey> hiddenTableColumnsLookupKeys = retrieveHiddenTableColumnsLookupKeys();
 
     switch (getRetrieverConnection().get(tableColumnsRetrievalStrategy)) {
       case data_dictionary_all:
@@ -106,7 +104,7 @@ private void createTableColumn(
       final MetadataResultSet results,
       final NamedObjectList<MutableTable> allTables,
       final InclusionRuleFilter<Column> columnFilter,
-      final Set<List<String>> hiddenTableColumnsLookupKeys) {
+      final Set<NamedObjectKey> hiddenTableColumnsLookupKeys) {
     // Get the "COLUMN_DEF" value first as it the Oracle drivers
     // don't handle it properly otherwise.
     // https://community.oracle.com/message/5940745#5940745
@@ -148,9 +146,7 @@ private void createTableColumn(
       final boolean isGenerated = results.getBoolean("IS_GENERATEDCOLUMN");
       final String remarks = results.getString("REMARKS");
 
-      final List<String> lookupKey =
-          Arrays.asList(columnCatalogName, schemaName, tableName, columnName);
-      final boolean isHidden = hiddenTableColumnsLookupKeys.contains(lookupKey);
+      final boolean isHidden = hiddenTableColumnsLookupKeys.contains(column.key());
 
       column.setOrdinalPosition(ordinalPosition);
       column.setColumnDataType(
@@ -202,9 +198,9 @@ private MutableColumn lookupOrCreateTableColumn(
     return column;
   }
 
-  private Set<List<String>> retrieveHiddenTableColumnsLookupKeys() throws SQLException {
+  private Set<NamedObjectKey> retrieveHiddenTableColumnsLookupKeys() throws SQLException {
 
-    final Set<List<String>> hiddenTableColumnsLookupKeys = new HashSet<>();
+    final Set<NamedObjectKey> hiddenTableColumnsLookupKeys = new HashSet<>();
 
     final InformationSchemaViews informationSchemaViews =
         getRetrieverConnection().getInformationSchemaViews();
@@ -230,8 +226,8 @@ private Set<List<String>> retrieveHiddenTableColumnsLookupKeys() throws SQLExcep
                 "Retrieving hidden column <%s.%s.%s.%s>",
                 catalogName, schemaName, tableName, columnName));
 
-        final List<String> lookupKey =
-            Arrays.asList(catalogName, schemaName, tableName, columnName);
+        final NamedObjectKey lookupKey =
+            new NamedObjectKey(catalogName, schemaName, tableName, columnName);
         hiddenTableColumnsLookupKeys.add(lookupKey);
       }
     }
@@ -242,7 +238,7 @@ private Set<List<String>> retrieveHiddenTableColumnsLookupKeys() throws SQLExcep
   private void retrieveTableColumnsFromDataDictionary(
       final NamedObjectList<MutableTable> allTables,
       final InclusionRuleFilter<Column> columnFilter,
-      final Set<List<String>> hiddenTableColumnsLookupKeys)
+      final Set<NamedObjectKey> hiddenTableColumnsLookupKeys)
       throws SQLException {
     final InformationSchemaViews informationSchemaViews =
         getRetrieverConnection().getInformationSchemaViews();
@@ -262,7 +258,7 @@ private void retrieveTableColumnsFromDataDictionary(
   private void retrieveTableColumnsFromMetadata(
       final NamedObjectList<MutableTable> allTables,
       final InclusionRuleFilter<Column> columnFilter,
-      final Set<List<String>> hiddenTableColumnsLookupKeys)
+      final Set<NamedObjectKey> hiddenTableColumnsLookupKeys)
       throws WrappedSQLException {
     for (final MutableTable table : allTables) {
       LOGGER.log(Level.FINE, "Retrieving table columns for " + table);

schemacrawler-dbtest/src/test/resources/testMariaDBWithConnection.8.txt

@@ -3,10 +3,10 @@
 System Information
 ========================================================================
 
-generated by                              SchemaCrawler 16.16.6         
-generated on                              2022-01-07 18:24:05           
+generated by                              SchemaCrawler 16.16.11        
+generated on                              2022-02-11 19:12:04           
 database version                          MariaDB 10.3.6-MariaDB-1:10.3.6+maria~jessie
-driver version                            MariaDB Connector/J 2.7.4     
+driver version                            MariaDB Connector/J 2.7.5     
 
 
 
@@ -220,6 +220,15 @@ Id                                                   [unique constraint]
 
 
 
+books.x_customers                                                [table]
+------------------------------------------------------------------------
+  CUSTOMERID                        INT NOT NULL                
+  CUSTOMER_NAME                     VARCHAR(80)                 
+  SOCIAL_SECURITY_NUMBER            CHAR(9)                     
+                                    hidden                      
+
+
+
 books.x_employees                                                [table]
 ------------------------------------------------------------------------
   EMPLOYEEID                        INT NOT NULL                
@@ -707,9 +716,9 @@ uses local files                          false
 
 JDBC Driver Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
-connection url                            jdbc:mariadb://localhost:55070/books
+connection url                            jdbc:mariadb://localhost:57679/books
 driver name                               MariaDB Connector/J           
-driver version                            2.7.4                         
+driver version                            2.7.5                         
 driver class name                         org.mariadb.jdbc.Driver       
 is JDBC compliant                         true                          
 supported JDBC version                    4.2                           

schemacrawler-dbtest/src/test/resources/testMariaDBWithConnection.LTE.txt

@@ -3,10 +3,10 @@
 System Information
 ========================================================================
 
-generated by                              SchemaCrawler 16.16.6         
-generated on                              2022-01-07 18:23:00           
+generated by                              SchemaCrawler 16.16.11        
+generated on                              2022-02-11 19:22:31           
 database version                          MariaDB 10.3.6-MariaDB-1:10.3.6+maria~jessie
-driver version                            MariaDB Connector/J 2.7.4     
+driver version                            MariaDB Connector/J 2.7.5     
 
 
 
@@ -220,6 +220,15 @@ Id                                                   [unique constraint]
 
 
 
+books.x_customers                                                [table]
+------------------------------------------------------------------------
+  CUSTOMERID                        INT NOT NULL                
+  CUSTOMER_NAME                     VARCHAR(80)                 
+  SOCIAL_SECURITY_NUMBER            CHAR(9)                     
+                                    hidden                      
+
+
+
 books.x_employees                                                [table]
 ------------------------------------------------------------------------
   EMPLOYEEID                        INT NOT NULL                
@@ -708,9 +717,9 @@ uses local files                          false
 
 JDBC Driver Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
-connection url                            jdbc:mariadb://localhost:54995/books
+connection url                            jdbc:mariadb://localhost:54930/books
 driver name                               MariaDB Connector/J           
-driver version                            2.7.4                         
+driver version                            2.7.5                         
 driver class name                         org.mariadb.jdbc.Driver       
 is JDBC compliant                         true                          
 supported JDBC version                    4.2                           

schemacrawler-mysql/src/main/resources/mysql.information_schema/ADDITIONAL_COLUMN_ATTRIBUTES.sql

@@ -7,6 +7,7 @@ SELECT
   CHARACTER_SET_NAME,
   COLLATION_NAME,
   COLUMN_COMMENT,
-  GENERATION_EXPRESSION
+  GENERATION_EXPRESSION,
+  EXTRA
 FROM
   INFORMATION_SCHEMA.COLUMNS

schemacrawler-mysql/src/main/resources/mysql.information_schema/EXT_HIDDEN_TABLE_COLUMNS.sql

@@ -0,0 +1,11 @@
+SELECT
+  TABLE_SCHEMA AS TABLE_CATALOG,
+  NULL AS TABLE_SCHEMA,
+  TABLE_NAME,
+  COLUMN_NAME,
+  COLUMN_COMMENT,
+  EXTRA
+FROM
+  INFORMATION_SCHEMA.COLUMNS
+WHERE
+  UPPER(EXTRA) = 'INVISIBLE'

schemacrawler-mysql/src/test/java/schemacrawler/integration/test/BundledDistributionTest.java

@@ -27,43 +27,39 @@
 */
 package schemacrawler.integration.test;
 
-
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 
 import java.sql.Connection;
 
 import org.junit.jupiter.api.Test;
+
 import schemacrawler.tools.databaseconnector.DatabaseConnector;
 import schemacrawler.tools.databaseconnector.DatabaseConnectorRegistry;
 
-public class BundledDistributionTest
-{
+public class BundledDistributionTest {
 
   @Test
-  public void testInformationSchema_mysql()
-    throws Exception
-  {
+  public void testInformationSchema_mysql() throws Exception {
 
     final Connection connection = null;
     final DatabaseConnectorRegistry registry =
-      DatabaseConnectorRegistry.getDatabaseConnectorRegistry();
+        DatabaseConnectorRegistry.getDatabaseConnectorRegistry();
     final DatabaseConnector databaseSystemIdentifier =
-      registry.findDatabaseConnectorFromDatabaseSystemIdentifier("mysql");
-    assertThat(databaseSystemIdentifier
-                 .getSchemaRetrievalOptionsBuilder(connection)
-                 .toOptions()
-                 .getInformationSchemaViews()
-                 .size(), is(12));
+        registry.findDatabaseConnectorFromDatabaseSystemIdentifier("mysql");
+    assertThat(
+        databaseSystemIdentifier
+            .getSchemaRetrievalOptionsBuilder(connection)
+            .toOptions()
+            .getInformationSchemaViews()
+            .size(),
+        is(13));
   }
 
   @Test
-  public void testPlugin_mysql()
-    throws Exception
-  {
+  public void testPlugin_mysql() throws Exception {
     final DatabaseConnectorRegistry registry =
-      DatabaseConnectorRegistry.getDatabaseConnectorRegistry();
+        DatabaseConnectorRegistry.getDatabaseConnectorRegistry();
     assertThat(registry.hasDatabaseSystemIdentifier("mysql"), is(true));
   }
-
 }

schemacrawler-mysql/src/test/resources/testMySQLWithConnection.8.txt

@@ -3,8 +3,8 @@
 System Information
 ========================================================================
 
-generated by                              SchemaCrawler 16.16.9         
-generated on                              2022-01-17 20:27:45           
+generated by                              SchemaCrawler 16.16.11        
+generated on                              2022-02-11 18:41:06           
 database version                          MySQL 8.0.27                  
 driver version                            MySQL Connector/J mysql-connector-java-8.0.28 (Revision: 7ff2161da3899f379fb3171b6538b191b1c5c7e2)
 
@@ -225,6 +225,15 @@ Id                                                   [unique constraint]
 
 
 
+books.x_customers                                                [table]
+------------------------------------------------------------------------
+  CUSTOMERID                        INT NOT NULL                
+  CUSTOMER_NAME                     VARCHAR(80)                 
+  SOCIAL_SECURITY_NUMBER            CHAR(9)                     
+                                    hidden                      
+
+
+
 books.x_employees                                                [table]
 ------------------------------------------------------------------------
   EMPLOYEEID                        INT NOT NULL                
@@ -530,8 +539,8 @@ System Information
 Database Server Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
 database                                  books                         
-hostname                                  dee805036a9a                  
-server_uuid                               e0e0875c-77d3-11ec-9986-0242ac110003
+hostname                                  12be6e6613de                  
+server_uuid                               220abd3a-8b6a-11ec-9ee8-0242ac110003
 
 
 Database Information
@@ -711,7 +720,7 @@ uses local files                          false
 
 JDBC Driver Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
-connection url                            jdbc:mysql://localhost:64859/books
+connection url                            jdbc:mysql://localhost:61893/books
 driver name                               MySQL Connector/J             
 driver version                            mysql-connector-java-8.0.28 (Revision: 7ff2161da3899f379fb3171b6538b191b1c5c7e2)
 driver class name                         com.mysql.cj.jdbc.Driver      
@@ -1329,7 +1338,7 @@ populateInsertRowWithDefaultValues                     [driver property]
 port                                                   [driver property]
   Port number of MySQL Server
   not required
-  value                             64859                       
+  value                             61893                       
 
 prepStmtCacheSize                                      [driver property]
   If prepared statement caching is enabled, how many prepared statements should be cached?

schemacrawler-mysql/src/test/resources/testMySQLWithConnection.LTE.txt

@@ -3,8 +3,8 @@
 System Information
 ========================================================================
 
-generated by                              SchemaCrawler 16.16.9         
-generated on                              2022-01-17 20:29:36           
+generated by                              SchemaCrawler 16.16.11        
+generated on                              2022-02-11 18:45:22           
 database version                          MySQL 8.0.27                  
 driver version                            MySQL Connector/J mysql-connector-java-8.0.28 (Revision: 7ff2161da3899f379fb3171b6538b191b1c5c7e2)
 
@@ -225,6 +225,15 @@ Id                                                   [unique constraint]
 
 
 
+books.x_customers                                                [table]
+------------------------------------------------------------------------
+  CUSTOMERID                        INT NOT NULL                
+  CUSTOMER_NAME                     VARCHAR(80)                 
+  SOCIAL_SECURITY_NUMBER            CHAR(9)                     
+                                    hidden                      
+
+
+
 books.x_employees                                                [table]
 ------------------------------------------------------------------------
   EMPLOYEEID                        INT NOT NULL                
@@ -530,8 +539,8 @@ System Information
 Database Server Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
 database                                  books                         
-hostname                                  b0fb43ac8762                  
-server_uuid                               248ca7d5-77d4-11ec-a698-0242ac110003
+hostname                                  1db1803dd232                  
+server_uuid                               bababd00-8b6a-11ec-853e-0242ac110003
 
 
 Database Information
@@ -712,7 +721,7 @@ uses local files                          false
 
 JDBC Driver Information
 -=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
-connection url                            jdbc:mysql://localhost:57747/books
+connection url                            jdbc:mysql://localhost:64050/books
 driver name                               MySQL Connector/J             
 driver version                            mysql-connector-java-8.0.28 (Revision: 7ff2161da3899f379fb3171b6538b191b1c5c7e2)
 driver class name                         com.mysql.cj.jdbc.Driver      
@@ -1330,7 +1339,7 @@ populateInsertRowWithDefaultValues                     [driver property]
 port                                                   [driver property]
   Port number of MySQL Server
   not required
-  value                             57747                       
+  value                             64050                       
 
 prepStmtCacheSize                                      [driver property]
   If prepared statement caching is enabled, how many prepared statements should be cached?

schemacrawler-testdb/src/main/resources/db/books/36_hidden_columns_A.sql

@@ -1,5 +1,5 @@
 -- Hidden columns
--- Oracle syntax
+-- Oracle and MySQL syntax
 CREATE TABLE X_CUSTOMERS
 (
   CUSTOMERID INTEGER NOT NULL,

schemacrawler-testdb/src/main/resources/mysql.scripts.txt

@@ -15,7 +15,7 @@
 ;,/db/books/33_temp_tables_B.sql
 ;,/db/books/34_indexes_A.sql
 ;,/db/books/35_generated_columns_C.sql
-#,/db/books/36_hidden_columns_A.sql
+;,/db/books/36_hidden_columns_A.sql
 ;,/db/books/40_procedures_B.sql
 ;,/db/books/41_functions_B.sql
 ;,/db/books/50_triggers_A.sql

schemacrawler-website/src/site/markdown/changes-report.md

@@ -3,6 +3,12 @@
 SchemaCrawler release notes.
 
 
+<a name="a16.16.12"></a>
+## Release 16.16.12 - 2022-02-20
+
+- Add support for hidden or invisible columns in MySQL
+
+
 <a name="a16.16.11"></a>
 ## Release 16.16.11 - 2022-02-05