Merge #59302

59302: sql: code generation for missing pg_catalog columns  r=RichardJCai a=mnovelodou

Previously, keeping up with postgres by adding the same columns in pg_catalog
was a manual process
This was inadequate because from version to version the list of missing
columns might increase a lot
To address this, this patch closes the gap between postgres and cockroachdb
by adding the missing columns automatically with nil values

Release note: None

Fixes: #58001

Co-authored-by: MiguelNovelo <miguel.novelo@digitalonus.com>

pkg/cmd/generate-postgres-metadata-tables/main.go

@@ -57,6 +57,10 @@ func main() {
  panic(err)
  }
  pgCatalogFile.PGMetadata.AddColumnMetadata(table, column, dataType, dataTypeOid)
+ columnType := pgCatalogFile.PGMetadata[table][column]
+ if !columnType.IsImplemented() {
+ pgCatalogFile.AddUnimplementedType(columnType)
+ }
  }
 
  pgCatalogFile.Save(os.Stdout)

pkg/sql/pg_metadata_diff.go

@@ -67,8 +67,9 @@ type PGMetadataTables map[string]PGMetadataColumns
 // PGMetadataFile is used to export pg_catalog from postgres and store the representation of this structure as a
 // json file
 type PGMetadataFile struct {
- PGVersion string `json:"pgVersion"`
- PGMetadata PGMetadataTables `json:"pgMetadata"`
+ PGVersion string `json:"pgVersion"`
+ PGMetadata PGMetadataTables `json:"pgMetadata"`
+ UnimplementedTypes map[oid.Oid]string `json:"unimplementedTypes"`
 }
 
 func (p PGMetadataTables) addColumn(tableName, columnName string, column *PGMetadataColumnType) {
@@ -215,6 +216,50 @@ func (p PGMetadataTables) getUnimplementedTables(source PGMetadataTables) PGMeta
  return notImplemented
 }
 
+// getUnimplementedColumns is used by diffs as it might not be in sync with
+// already implemented columns.
+func (p PGMetadataTables) getUnimplementedColumns(target PGMetadataTables) PGMetadataTables {
+ unimplementedColumns := make(PGMetadataTables)
+ for tableName, columns := range p {
+ for columnName, columnType := range columns {
+ if columnType != nil {
+ // dataType mismatch (Not a new column).
+ continue
+ }
+ sourceType := target[tableName][columnName]
+ typeOid := oid.Oid(sourceType.Oid)
+ if _, ok := types.OidToType[typeOid]; !ok || typeOid == oid.T_anyarray {
+ // can't implement this column due to missing type.
+ continue
+ }
+ unimplementedColumns.AddColumnMetadata(tableName, columnName, sourceType.DataType, sourceType.Oid)
+ }
+ }
+ return unimplementedColumns
+}
+
+// removeImplementedColumns removes diff columns that are marked as expected
+// diff (or unimplemented column) but is already implemented in CRDB.
+func (p PGMetadataTables) removeImplementedColumns(source PGMetadataTables) {
+ for tableName, columns := range source {
+ pColumns, exists := p[tableName]
+ if !exists {
+ continue
+ }
+ for columnName := range columns {
+ columnType, exists := pColumns[columnName]
+ if !exists {
+ continue
+ }
+ if columnType != nil {
+ continue
+ }
+
+ delete(pColumns, columnName)
+ }
+ }
+}
+
 // getUnimplementedTypes verifies that all the types are implemented in cockroach db.
 func (c PGMetadataColumns) getUnimplementedTypes() map[oid.Oid]string {
  unimplemented := make(map[oid.Oid]string)
@@ -227,3 +272,22 @@ func (c PGMetadataColumns) getUnimplementedTypes() map[oid.Oid]string {
 
  return unimplemented
 }
+
+// AddUnimplementedType reports a type that is not implemented in cockroachdb.
+func (f *PGMetadataFile) AddUnimplementedType(columnType *PGMetadataColumnType) {
+ typeOid := oid.Oid(columnType.Oid)
+ if f.UnimplementedTypes == nil {
+ f.UnimplementedTypes = make(map[oid.Oid]string)
+ }
+
+ f.UnimplementedTypes[typeOid] = columnType.DataType
+}
+
+// IsImplemented determines whether the type is implemented or not in
+// cockroachdb.
+func (t *PGMetadataColumnType) IsImplemented() bool {
+ typeOid := oid.Oid(t.Oid)
+ _, ok := types.OidToType[typeOid]
+ // Cannot use type oid.T_anyarray in CREATE TABLE
+ return ok && typeOid != oid.T_anyarray
+}