[Feat](nereids) support generated column

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4

@@ -115,6 +115,7 @@ AT: 'AT';
 AUTHORS: 'AUTHORS';
 AUTO: 'AUTO';
 AUTO_INCREMENT: 'AUTO_INCREMENT';
+ALWAYS: 'ALWAYS';
 BACKEND: 'BACKEND';
 BACKENDS: 'BACKENDS';
 BACKUP: 'BACKUP';
@@ -276,6 +277,7 @@ FRONTENDS: 'FRONTENDS';
 FULL: 'FULL';
 FUNCTION: 'FUNCTION';
 FUNCTIONS: 'FUNCTIONS';
+GENERATED: 'GENERATED';
 GENERIC: 'GENERIC';
 GLOBAL: 'GLOBAL';
 GRANT: 'GRANT';

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4

@@ -585,14 +585,15 @@ columnDef
     : colName=identifier type=dataType
         KEY?
         (aggType=aggTypeDef)?
+        ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)?
         ((NOT)? nullable=NULL)?
         (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
         (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | stringValue=STRING_LITERAL
            | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
         (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
         (COMMENT comment=STRING_LITERAL)?
     ;
-    
+
 indexDefs
     : indexes+=indexDef (COMMA indexes+=indexDef)*
     ;
@@ -1028,6 +1029,7 @@ nonReserved
     | AGG_STATE
     | AGGREGATE
     | ALIAS
+    | ALWAYS
     | ANALYZED
     | ARRAY
     | ARRAY_RANGE
@@ -1142,6 +1144,7 @@ nonReserved
     | FREE
     | FRONTENDS
     | FUNCTION
+    | GENERATED
     | GENERIC
     | GLOBAL
     | GRAPH

fe/fe-core/src/main/cup/sql_parser.cup

@@ -43,6 +43,7 @@ import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.GeneratedColumnInfo;
 import org.apache.doris.catalog.MapType;
 import org.apache.doris.catalog.StructField;
 import org.apache.doris.catalog.StructType;
@@ -62,6 +63,7 @@ import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import java_cup.runtime.Symbol;
@@ -261,6 +263,7 @@ terminal String
     KW_ALIAS,
     KW_ALL,
     KW_ALTER,
+    KW_ALWAYS,
     KW_ANALYZE,
     KW_AND,
     KW_ANTI,
@@ -408,6 +411,7 @@ terminal String
     KW_FULL,
     KW_FUNCTION,
     KW_FUNCTIONS,
+    KW_GENERATED,
     KW_GENERIC,
     KW_GLOBAL,
     KW_GRANT,
@@ -1004,6 +1008,9 @@ nonterminal String stage_name;
 nonterminal StageAndPattern stage_and_pattern;
 nonterminal List<Expr> copy_select_expr_list;
 
+//genearted column
+nonterminal Boolean opt_generated_always;
+
 precedence nonassoc COMMA;
 precedence nonassoc STRING_LITERAL;
 precedence nonassoc KW_COLUMNS;
@@ -3968,6 +3975,11 @@ column_definition ::=
         ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, aggType, nullable_type, autoIncInitValue, defaultValue, comment);
         RESULT = columnDef;
     :}
+    | ident:columnName type_def:typeDef opt_is_key:isKey opt_generated_always KW_AS LPAREN expr:expr RPAREN opt_nullable_type:nullable_type opt_comment:comment
+    {:
+        ColumnDef columnDef =  new ColumnDef(columnName, typeDef, isKey, nullable_type, comment, Optional.of(new GeneratedColumnInfo(null, expr)));
+        RESULT = columnDef;
+    :}
     ;
 
 index_definition ::=
@@ -7942,7 +7954,16 @@ opt_work ::=
         RESULT = null;
     :}
     ;
-
+opt_generated_always ::=
+    /* empty */
+    {:
+        RESULT = null;
+    :}
+    | KW_GENERATED KW_ALWAYS
+    {:
+        RESULT = null;
+    :}
+    ;
 opt_chain ::=
     {:
         RESULT = null;
@@ -7982,6 +8003,8 @@ type_func_name_keyword ::=
 keyword ::=
     KW_AFTER:id
     {: RESULT = id; :}
+    | KW_ALWAYS:id
+    {: RESULT = id; :}
     | KW_AGGREGATE:id
     {: RESULT = id; :}
     | KW_ALIAS:id
@@ -8142,6 +8165,8 @@ keyword ::=
     {: RESULT = id; :}
     | KW_GLOBAL:id
     {: RESULT = id; :}
+    | KW_GENERATED:id
+    {: RESULT = id; :}
     | KW_GENERIC:id
     {: RESULT = id; :}
     | KW_GRAPH:id

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -28,6 +28,7 @@
 import org.apache.doris.analysis.CreateMaterializedViewStmt;
 import org.apache.doris.analysis.DropColumnClause;
 import org.apache.doris.analysis.DropIndexClause;
+import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.IndexDef;
 import org.apache.doris.analysis.ModifyColumnClause;
 import org.apache.doris.analysis.ModifyTablePropertiesClause;
@@ -42,6 +43,7 @@
 import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.EnvFactory;
+import org.apache.doris.catalog.GeneratedColumnInfo;
 import org.apache.doris.catalog.HashDistributionInfo;
 import org.apache.doris.catalog.Index;
 import org.apache.doris.catalog.KeysType;
@@ -377,6 +379,23 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
             }
         }
 
+        // generated column check
+        Map<String, Column> nameToColumn = new HashMap<>();
+        for (Column c : indexSchemaMap.get(baseIndexId)) {
+            nameToColumn.put(c.getName(), c);
+        }
+        if (null == targetIndexName) {
+            if (nameToColumn.containsKey(dropColName)) {
+                Column column = nameToColumn.get(dropColName);
+                Set<String> generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis();
+                if (!generatedColumnsThatReferToThis.isEmpty()) {
+                    throw new DdlException(
+                            "Column '" + dropColName + "' has a generated column dependency on :"
+                                    + generatedColumnsThatReferToThis);
+                }
+            }
+        }
+
         Iterator<Index> it = indexes.iterator();
         while (it.hasNext()) {
             Index index = it.next();
@@ -403,6 +422,8 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
                 if (column.getName().equalsIgnoreCase(dropColName)) {
                     baseIter.remove();
                     found = true;
+                    // find generated column referred column
+                    removeColumnWhenDropGeneratedColumn(column, nameToColumn);
                     break;
                 }
             }
@@ -528,6 +549,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
                                         Map<Long, LinkedList<Column>> indexSchemaMap) throws DdlException {
         Column modColumn = alterClause.getColumn();
         boolean lightSchemaChange = false;
+
         if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
             if (modColumn.isKey() && null != modColumn.getAggregationType()) {
                 throw new DdlException("Can not assign aggregation method on key column: " + modColumn.getName());
@@ -784,12 +806,15 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol
         if (targetIndexName == null) {
             targetIndexName = baseIndexName;
         }
-
         long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
 
         LinkedList<Column> newSchema = new LinkedList<Column>();
         List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
-
+        // When rollup is specified, there is no need to check the order of generated columns.
+        // When rollup is not specified and the order of baseIndex needs to be modified, the order needs to be checked.
+        if (alterClause.getRollupName() == null) {
+            checkOrder(targetIndexSchema, orderedColNames);
+        }
         // check and create new ordered column list
         Set<String> colNameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
         for (String colName : orderedColNames) {
@@ -1004,6 +1029,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
             }
         }
 
+        if (newColumn.getGeneratedColumnInfo() != null) {
+            throw new DdlException("Not supporting alter table add generated columns.");
+        }
+
         /*
          * add new column to indexes.
          * UNIQUE:
@@ -3194,4 +3223,61 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List<AlterCl
 
         return false;
     }
+
+    private void removeColumnWhenDropGeneratedColumn(Column dropColumn, Map<String, Column> nameToColumn) {
+        GeneratedColumnInfo generatedColumnInfo = dropColumn.getGeneratedColumnInfo();
+        if (generatedColumnInfo == null) {
+            return;
+        }
+        String dropColName = dropColumn.getName();
+        Expr expr = generatedColumnInfo.getExpr();
+        Set<Expr> slotRefsInGeneratedExpr = new HashSet<>();
+        expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr);
+        for (Expr slotRef : slotRefsInGeneratedExpr) {
+            String name = ((SlotRef) slotRef).getColumnName();
+            if (!nameToColumn.containsKey(name)) {
+                continue;
+            }
+            Column c = nameToColumn.get(name);
+            Set<String> sets = c.getGeneratedColumnsThatReferToThis();
+            sets.remove(dropColName);
+        }
+    }
+
+    private void checkOrder(List<Column> targetIndexSchema, List<String> orderedColNames) throws DdlException {
+        Set<String> nameSet = new HashSet<>();
+        for (Column column : targetIndexSchema) {
+            if (column.isVisible() && null == column.getGeneratedColumnInfo()) {
+                nameSet.add(column.getName());
+            }
+        }
+        for (String colName : orderedColNames) {
+            Column oneCol = null;
+            for (Column column : targetIndexSchema) {
+                if (column.getName().equalsIgnoreCase(colName) && column.isVisible()) {
+                    oneCol = column;
+                    break;
+                }
+            }
+            if (oneCol == null) {
+                throw new DdlException("Column[" + colName + "] not exists");
+            }
+            if (null == oneCol.getGeneratedColumnInfo()) {
+                continue;
+            }
+            Expr expr = oneCol.getGeneratedColumnInfo().getExpr();
+            Set<Expr> slotRefsInGeneratedExpr = new HashSet<>();
+            expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr);
+            for (Expr slotRef : slotRefsInGeneratedExpr) {
+                String slotName = ((SlotRef) slotRef).getColumnName();
+                if (!nameSet.contains(slotName)) {
+                    throw new DdlException("The specified column order is incorrect, `" + colName
+                            + "` should come after `" + slotName
+                            + "`, because both of them are generated columns, and `"
+                            + colName + "` refers to `" + slotName + "`.");
+                }
+            }
+            nameSet.add(colName);
+        }
+    }
 }