From 00a96c93ee9891cb3ebd3510c5244a3e3cc5660c Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Fri, 29 Nov 2024 06:37:36 -0500 Subject: [PATCH 01/29] Update existing ports of Oracle MySQL to set defaults in constructors, so Test.* can be generated/removed. Add Dart port. --- sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts | 98 ++-- sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts | 44 +- sql/mysql/Oracle/Antlr4ng/Test.ts | 289 ----------- sql/mysql/Oracle/CSharp/MySQLLexerBase.cs | 22 +- sql/mysql/Oracle/CSharp/MySQLParserBase.cs | 33 ++ sql/mysql/Oracle/CSharp/StackQueue.cs | 141 ------ sql/mysql/Oracle/CSharp/Test.cs | 483 ------------------- sql/mysql/Oracle/Dart/MySQLLexerBase.dart | 387 +++++++++++++++ sql/mysql/Oracle/Dart/MySQLParserBase.dart | 82 ++++ sql/mysql/Oracle/Java/MySQLLexerBase.java | 10 +- sql/mysql/Oracle/desc.xml | 2 +- 11 files changed, 611 insertions(+), 980 deletions(-) delete mode 100644 sql/mysql/Oracle/Antlr4ng/Test.ts delete mode 100644 sql/mysql/Oracle/CSharp/StackQueue.cs delete mode 100644 sql/mysql/Oracle/CSharp/Test.cs create mode 100644 sql/mysql/Oracle/Dart/MySQLLexerBase.dart create mode 100644 sql/mysql/Oracle/Dart/MySQLParserBase.dart diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts index 036674d09e..3a418acefb 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts @@ -5,7 +5,7 @@ /* eslint-disable no-underscore-dangle */ /* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ -import { Lexer, Token } from "antlr4ng"; +import { CharStream, Lexer, Token } from "antlr4ng"; import { MySQLLexer } from "./MySQLLexer.js"; /** SQL modes that control parsing behavior. */ @@ -41,6 +41,12 @@ export abstract class MySQLLexerBase extends Lexer { static #unsignedLongLongString = "18446744073709551615"; static #unsignedLongLongLength = 20; + constructor(input: CharStream) { + super(input); + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + /** * Determines if the given SQL mode is currently active in the lexer. * @@ -313,226 +319,226 @@ export abstract class MySQLLexerBase extends Lexer { public doLogicalOr(): void { - this.type = this.isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; + this.type = this.isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; } public doIntNumber(): void { - this.type = this.determineNumericType(this.text); + this.type = this.determineNumericType(this.text); } public doAdddate(): void { - this.type = this.determineFunction(MySQLLexer.ADDDATE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.ADDDATE_SYMBOL); } public doBitAnd(): void { - this.type = this.determineFunction(MySQLLexer.BIT_AND_SYMBOL); + this.type = this.determineFunction(MySQLLexer.BIT_AND_SYMBOL); } public doBitOr(): void { - this.type = this.determineFunction(MySQLLexer.BIT_OR_SYMBOL); + this.type = this.determineFunction(MySQLLexer.BIT_OR_SYMBOL); } public doBitXor(): void { - this.type = this.determineFunction(MySQLLexer.BIT_XOR_SYMBOL); + this.type = this.determineFunction(MySQLLexer.BIT_XOR_SYMBOL); } public doCast(): void { - this.type = this.determineFunction(MySQLLexer.CAST_SYMBOL); + this.type = this.determineFunction(MySQLLexer.CAST_SYMBOL); } public doCount(): void { - this.type = this.determineFunction(MySQLLexer.COUNT_SYMBOL); + this.type = this.determineFunction(MySQLLexer.COUNT_SYMBOL); } public doCurdate(): void { - this.type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); } public doCurrentDate(): void { - this.type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); } public doCurrentTime(): void { - this.type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + this.type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); } public doCurtime(): void { - this.type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + this.type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); } public doDateAdd(): void { - this.type = this.determineFunction(MySQLLexer.DATE_ADD_SYMBOL); + this.type = this.determineFunction(MySQLLexer.DATE_ADD_SYMBOL); } public doDateSub(): void { - this.type = this.determineFunction(MySQLLexer.DATE_SUB_SYMBOL); + this.type = this.determineFunction(MySQLLexer.DATE_SUB_SYMBOL); } public doExtract(): void { - this.type = this.determineFunction(MySQLLexer.EXTRACT_SYMBOL); + this.type = this.determineFunction(MySQLLexer.EXTRACT_SYMBOL); } public doGroupConcat(): void { - this.type = this.determineFunction(MySQLLexer.GROUP_CONCAT_SYMBOL); + this.type = this.determineFunction(MySQLLexer.GROUP_CONCAT_SYMBOL); } public doMax(): void { - this.type = this.determineFunction(MySQLLexer.MAX_SYMBOL); + this.type = this.determineFunction(MySQLLexer.MAX_SYMBOL); } public doMid(): void { - this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doMin(): void { - this.type = this.determineFunction(MySQLLexer.MIN_SYMBOL); + this.type = this.determineFunction(MySQLLexer.MIN_SYMBOL); } public doNot(): void { - this.type = this.isSqlModeActive(SqlMode.HighNotPrecedence) ? MySQLLexer.NOT2_SYMBOL: MySQLLexer.NOT_SYMBOL; + this.type = this.isSqlModeActive(SqlMode.HighNotPrecedence) ? MySQLLexer.NOT2_SYMBOL: MySQLLexer.NOT_SYMBOL; } public doNow(): void { - this.type = this.determineFunction(MySQLLexer.NOW_SYMBOL); + this.type = this.determineFunction(MySQLLexer.NOW_SYMBOL); } public doPosition(): void { - this.type = this.determineFunction(MySQLLexer.POSITION_SYMBOL); + this.type = this.determineFunction(MySQLLexer.POSITION_SYMBOL); } public doSessionUser(): void { - this.type = this.determineFunction(MySQLLexer.USER_SYMBOL); + this.type = this.determineFunction(MySQLLexer.USER_SYMBOL); } public doStddevSamp(): void { - this.type = this.determineFunction(MySQLLexer.STDDEV_SAMP_SYMBOL); + this.type = this.determineFunction(MySQLLexer.STDDEV_SAMP_SYMBOL); } public doStddev(): void { - this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doStddevPop(): void { - this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doStd(): void { - this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this.type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doSubdate(): void { - this.type = this.determineFunction(MySQLLexer.SUBDATE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SUBDATE_SYMBOL); } public doSubstr(): void { - this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doSubstring(): void { - this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doSum(): void { - this.type = this.determineFunction(MySQLLexer.SUM_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SUM_SYMBOL); } public doSysdate(): void { - this.type = this.determineFunction(MySQLLexer.SYSDATE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.SYSDATE_SYMBOL); } public doSystemUser(): void { - this.type = this.determineFunction(MySQLLexer.USER_SYMBOL); + this.type = this.determineFunction(MySQLLexer.USER_SYMBOL); } public doTrim(): void { - this.type = this.determineFunction(MySQLLexer.TRIM_SYMBOL); + this.type = this.determineFunction(MySQLLexer.TRIM_SYMBOL); } public doVariance(): void { - this.type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); } public doVarPop(): void { - this.type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + this.type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); } public doVarSamp(): void { - this.type = this.determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); + this.type = this.determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); } public doUnderscoreCharset(): void { - this.type = this.checkCharset(this.text); + this.type = this.checkCharset(this.text); } public isVersionComment(): boolean { - return this.checkMySQLVersion(this.text); + return this.checkMySQLVersion(this.text); } public isBackTickQuotedId(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public isDoubleQuotedText(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public isSingleQuotedText(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public startInVersionComment(): void { - this.inVersionComment = true; + this.inVersionComment = true; } public endInVersionComment(): void { - this.inVersionComment = false; + this.inVersionComment = false; } public isInVersionComment(): boolean { - return this.inVersionComment; + return this.inVersionComment; } } diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts index 3c80d99e1c..cbe88f5bf5 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts @@ -2,8 +2,7 @@ * Copyright © 2024, Oracle and/or its affiliates */ -import { Parser } from "antlr4ng"; - +import { Parser, TokenStream } from "antlr4ng"; import { SqlMode } from "./MySQLLexerBase.js"; export abstract class MySQLParserBase extends Parser { @@ -15,6 +14,12 @@ export abstract class MySQLParserBase extends Parser { /** Enable Multi Language Extension support. */ public supportMle = true; + constructor(input: TokenStream) { + super(input); + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + /** * Determines if the given SQL mode is currently active in the lexer. * @@ -28,21 +33,48 @@ export abstract class MySQLParserBase extends Parser { public isPureIdentifier(): boolean { - return this.isSqlModeActive(SqlMode.AnsiQuotes); + return this.isSqlModeActive(SqlMode.AnsiQuotes); } public isTextStringLiteral(): boolean { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); + return !this.isSqlModeActive(SqlMode.AnsiQuotes); } public isStoredRoutineBody(): boolean { - return this.serverVersion >= 80032 && this.supportMle; + return this.serverVersion >= 80032 && this.supportMle; } public isSelectStatementWithInto(): boolean { - return this.serverVersion >= 80024 && this.serverVersion < 80031; + return this.serverVersion >= 80024 && this.serverVersion < 80031; + } + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public sqlModeFromString(modes: string): void { + this.sqlModes = new Set(); + + const parts = modes.toUpperCase().split(","); + parts.forEach((mode: string) => { + if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || + mode === "POSTGRESQL") { + this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); + } else if (mode === "ANSI_QUOTES") { + this.sqlModes.add(SqlMode.AnsiQuotes); + } else if (mode === "PIPES_AS_CONCAT") { + this.sqlModes.add(SqlMode.PipesAsConcat); + } else if (mode === "NO_BACKSLASH_ESCAPES") { + this.sqlModes.add(SqlMode.NoBackslashEscapes); + } else if (mode === "IGNORE_SPACE") { + this.sqlModes.add(SqlMode.IgnoreSpace); + } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { + this.sqlModes.add(SqlMode.HighNotPrecedence); + } + }); } } diff --git a/sql/mysql/Oracle/Antlr4ng/Test.ts b/sql/mysql/Oracle/Antlr4ng/Test.ts deleted file mode 100644 index b6384b4645..0000000000 --- a/sql/mysql/Oracle/Antlr4ng/Test.ts +++ /dev/null @@ -1,289 +0,0 @@ -// Generated from trgen 0.23.7 - -import { ATNSimulator } from 'antlr4ng'; -import { BaseErrorListener } from 'antlr4ng'; -import { CharStream } from 'antlr4ng'; -import { CommonTokenStream } from 'antlr4ng'; -import { ConsoleErrorListener } from 'antlr4ng'; -import { ErrorNode } from 'antlr4ng'; -//import { InputStream } from 'antlr4ng'; -import { Parser } from 'antlr4ng'; -import { ParserRuleContext } from 'antlr4ng'; -import { ParseTree } from 'antlr4ng'; -import { Recognizer } from 'antlr4ng'; -import { RecognitionException } from 'antlr4ng'; -import { TerminalNode } from 'antlr4ng'; -import { Token } from 'antlr4ng'; -import { Trees } from 'antlr4ng'; -import { escapeWhitespace } from 'antlr4ng'; -import { readFileSync } from 'fs'; -import { writeFileSync } from 'fs'; -import { openSync } from 'fs'; -import { readSync } from 'fs'; -import { writeSync } from 'fs'; -import { closeSync } from 'fs'; -import { readFile } from 'fs/promises' -import { isToken } from 'antlr4ng'; - -import { MySQLLexer } from './MySQLLexer.js'; -import { MySQLParser } from './MySQLParser.js'; - -import { StringBuilder, emptyString, joinString, formatString, isNullOrWhiteSpace } from 'typescript-string-operations'; -import { Timer, Time, TimerOptions } from 'timer-node'; - - -function getChar() { - let buffer = Buffer.alloc(1); - var xx = 0; - try { - xx = readSync(0, buffer, 0, 1, null); - } catch (err) { - } - if (xx === 0) { - return ''; - } - return buffer.toString('utf8'); -} - - -class MyErrorListener extends ConsoleErrorListener { - _quiet: boolean; - _tee: boolean; - _output: any; - had_error: boolean; - - constructor(quiet: boolean, tee: boolean, output: any) { - super(); - this._quiet = quiet; - this._tee = tee; - this._output = output; - this.had_error = false; - } - - syntaxError(recognizer: Recognizer | null, offendingSymbol: unknown, line: number, column: number, msg: string | null, e: RecognitionException | null): void { - this.had_error = true; - if (this._tee) { - writeSync(this._output, `line ${line}:${column} ${msg}\n`); - } - if (!this._quiet) { - console.error(`line ${line}:${column} ${msg}`); - } - } -} - -var tee = false; -var show_profile = false; -var show_tree = false; -var show_tokens = false; -var show_trace = false; -var error_code = 0; -var quiet = false; -var enc = 'utf8'; -var string_instance = 0; -var prefix = ''; -var inputs: string[] = []; -var is_fns: boolean[] = []; - -function splitLines(t: string) { return t.split(/\r\n|\r|\n/); } - -function main() { - for (let i = 2; i charSets = new HashSet(); // Used to check repertoires. protected bool inVersionComment = false; - private StackQueue pendingTokens = new StackQueue(); + private Queue pendingTokens = new Queue(); static string longString = "2147483647"; static int longLength = 10; @@ -55,13 +55,17 @@ public class MySQLLexerBase : Lexer { protected MySQLLexerBase(ICharStream input, TextWriter output, TextWriter errorOutput) - : base(input, output, errorOutput) + : base(input, output, errorOutput) { + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); } public MySQLLexerBase(ICharStream input) : base(input) { + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); } /** @@ -125,8 +129,9 @@ public override void Reset() public override IToken NextToken() { // First respond with pending tokens to the next token request, if there are any. - var pending = this.pendingTokens.DequeueBottom(); - if (pending != null) { + IToken pending; + var not_empty = this.pendingTokens.TryDequeue(out pending); + if (not_empty) { return pending; } @@ -134,10 +139,9 @@ public override IToken NextToken() // This might create additional tokens again. var next = base.NextToken(); - pending = this.pendingTokens.DequeueBottom(); - if (pending != null) { - this.pendingTokens.Push(next); - + not_empty = this.pendingTokens.TryDequeue(out pending); + if (not_empty) { + this.pendingTokens.Enqueue(next); return pending; } @@ -293,7 +297,7 @@ protected int checkCharset(string text) */ protected void emitDot() { - this.pendingTokens.Push(this.TokenFactory.Create(new Tuple(this, (ICharStream)this.InputStream), MySQLLexer.DOT_SYMBOL, + this.pendingTokens.Enqueue(this.TokenFactory.Create(new Tuple(this, (ICharStream)this.InputStream), MySQLLexer.DOT_SYMBOL, this.Text, this.Channel, this.TokenStartCharIndex, this.TokenStartCharIndex, this.Line, this.Column )); diff --git a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs index 25233e7520..7c7274eb11 100644 --- a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs @@ -17,6 +17,8 @@ public abstract class MySQLParserBase : Parser { protected MySQLParserBase(ITokenStream input, TextWriter output, TextWriter errorOutput) : base(input, output, errorOutput) { + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); } /** @@ -49,4 +51,35 @@ public bool isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public void sqlModeFromString(string modes) + { + this.sqlModes = new HashSet(); + + var parts = modes.ToUpper().Split(","); + foreach (var mode in parts) + { + if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || mode == "ORACLE" || + mode == "POSTGRESQL") { + this.sqlModes.Add(SqlMode.AnsiQuotes); + this.sqlModes.Add(SqlMode.PipesAsConcat); + this.sqlModes.Add(SqlMode.IgnoreSpace); + } else if (mode == "ANSI_QUOTES") { + this.sqlModes.Add(SqlMode.AnsiQuotes); + } else if (mode == "PIPES_AS_CONCAT") { + this.sqlModes.Add(SqlMode.PipesAsConcat); + } else if (mode == "NO_BACKSLASH_ESCAPES") { + this.sqlModes.Add(SqlMode.NoBackslashEscapes); + } else if (mode == "IGNORE_SPACE") { + this.sqlModes.Add(SqlMode.IgnoreSpace); + } else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40") { + this.sqlModes.Add(SqlMode.HighNotPrecedence); + } + } + } } diff --git a/sql/mysql/Oracle/CSharp/StackQueue.cs b/sql/mysql/Oracle/CSharp/StackQueue.cs deleted file mode 100644 index 6bc4e296dd..0000000000 --- a/sql/mysql/Oracle/CSharp/StackQueue.cs +++ /dev/null @@ -1,141 +0,0 @@ -using System.Collections; -using System.Collections.Generic; - -public class StackQueue : IEnumerable -{ - private readonly List _items; - - public StackQueue() - { - _items = new List(); - } - - public StackQueue(T value) - { - _items = new List(); - _items.Add(value); - } - - public StackQueue(StackQueue other) - { - _items = new List(); - _items.AddRange(other._items); - } - - public virtual int Size() - { - return _items.Count; - } - - public virtual int Count => _items.Count; - - public virtual T Pop() - { - if (_items.Count > 0) - { - var result = _items[_items.Count - 1]; - _items.RemoveAt(_items.Count - 1); - return result; - } - else return default; - } - - public virtual T this[int n] - { - get => PeekBottom(n); - set => _items[n] = value; - } - - public virtual T Peek() - { - return PeekTop(0); - } - - public bool Any() - { - return _items.Count > 0; - } - - public virtual T PeekTop(int n = 0) - { - if (_items.Count - n > 0) - { - int index = _items.Count - n - 1; - var result = _items[index]; - return result; - } - else return default; - } - - public virtual T PeekBottom(int n) - { - if (n >= 0 && n < _items.Count - 1) - { - var result = _items[n]; - return result; - } - else return default; - } - - public virtual void Push(T value) - { - _items.Add(value); - } - - public virtual void Push(IEnumerable collection) - { - foreach (T t in collection) - { - _items.Add(t); - } - } - - public virtual void PushMultiple(params T[] values) - { - int count = values.Length; - for (int i = 0; i < count; i++) _items.Add(values[i]); - } - - public virtual void EnqueueTop(T value) - { - Push(value); - } - - public virtual void EnqueueBottom(T value) - { - _items.Insert(0, value); - } - - public virtual T DequeueTop() - { - return Pop(); - } - - public virtual T DequeueBottom() - { - if (_items.Count > 0) - { - var result = _items[0]; - _items.RemoveAt(0); - return result; - } - else return default; - } - - public virtual bool Contains(T item) - { - return _items.Contains(item); - } - - public virtual System.Collections.Generic.IEnumerator GetEnumerator() - { - for (int i = _items.Count - 1; i >= 0; i--) - yield return _items[i]; - } - - IEnumerator IEnumerable.GetEnumerator() - { - for (int i = _items.Count - 1; i >= 0; i--) - yield return _items[i]; - } -} diff --git a/sql/mysql/Oracle/CSharp/Test.cs b/sql/mysql/Oracle/CSharp/Test.cs deleted file mode 100644 index 5c4636221c..0000000000 --- a/sql/mysql/Oracle/CSharp/Test.cs +++ /dev/null @@ -1,483 +0,0 @@ -// Generated from trgen 0.23.7 - -using Antlr4.Runtime; -using Antlr4.Runtime.Atn; -using Antlr4.Runtime.Tree; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -public class Program -{ - public static MySQLParser Parser { get; set; } - public static ErrorListener ParserErrorListener { get; set; } - public static MySQLLexer Lexer { get; set; } - public static ErrorListener LexerErrorListener { get; set; } - public static ITokenStream TokenStream { get; set; } - public static ICharStream CharStream { get; set; } - public static IParseTree Tree { get; set; } - public static List Trees { get; set; } - public static string StartSymbol { get; set; } = "queries"; - public static string Input { get; set; } - public static bool HeatMap { get; set; } = false; - public static void SetupParse2(string input, bool quiet = false) - { - ICharStream str = new AntlrInputStream(input); - CharStream = str; - var lexer = new MySQLLexer(str); - lexer.serverVersion = 80200; - lexer.sqlModeFromString("ANSI_QUOTES"); - Lexer = lexer; - CommonTokenStream tokens = null; - if (HeatMap) { - tokens = new ProfilingCommonTokenStream(lexer); - } - else { - tokens = new CommonTokenStream(lexer); - } - TokenStream = tokens; - var parser = new MyParser(tokens); - parser.serverVersion = lexer.serverVersion; - parser.sqlModes = lexer.sqlModes; - Parser = parser; - var listener_lexer = new ErrorListener(false, false, System.Console.Error); - var listener_parser = new ErrorListener(false, false, System.Console.Error); - LexerErrorListener = listener_lexer; - ParserErrorListener = listener_parser; - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - } - - public static IParseTree Parse2() - { - var tree = Parser.queries(); - Input = Lexer.InputStream.ToString(); - TokenStream = Parser.TokenStream; - Tree = tree; - return tree; - } - - public static List>> Parse3() - { - Parser.Profile = true; - var tree = Parser.queries(); - var decisions = Parser.ParseInfo.getDecisionInfo().Where(d => d.ambiguities.Any()).ToList(); - var result = new List>>(); - foreach (var decision in decisions) - { - var am = decision.ambiguities; - var trees = new List(); - foreach (AmbiguityInfo ai in am) - { - var parser_decision = ai.decision; - var parser_alts = ai.ambigAlts; - var parser_startIndex = ai.startIndex; - var parser_stopIndex = ai.stopIndex; - var p = Parser.RuleNames.Select((value, index) => new { value, index }) - .Where(pair => (pair.value == "queries")) - .Select(pair => pair.index).First(); - var parser_startRuleIndex = p; - var parser_trees = ((MyParser)Parser).getAllPossibleParseTrees( - parser_decision, - parser_alts, - parser_startIndex, - parser_stopIndex, - parser_startRuleIndex); - trees.AddRange(parser_trees); - } - result.Add(new Tuple>(decision.decision, trees)); - } - Input = Lexer.InputStream.ToString(); - TokenStream = Parser.TokenStream; - return result; - } - - public static bool AnyErrors() - { - return ParserErrorListener.had_error || LexerErrorListener.had_error; - } - - public static IParseTree Parse(string input) - { - ICharStream str = new AntlrInputStream(input); - CharStream = str; - var lexer = new MySQLLexer(str); - lexer.serverVersion = 80200; - lexer.sqlModeFromString("ANSI_QUOTES"); - Lexer = lexer; - CommonTokenStream tokens = null; - if (show_hit) { - tokens = new ProfilingCommonTokenStream(lexer); - } - else { - tokens = new CommonTokenStream(lexer); - } - TokenStream = tokens; - var parser = new MySQLParser(tokens); - parser.serverVersion = lexer.serverVersion; - parser.sqlModes = lexer.sqlModes; - Parser = parser; - var listener_lexer = new ErrorListener(false, false, System.Console.Error); - var listener_parser = new ErrorListener(false, false, System.Console.Error); - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - var tree = parser.queries(); - Input = lexer.InputStream.ToString(); - TokenStream = parser.TokenStream; - Tree = tree; - return tree; - } - - static bool tee = false; - static bool show_diagnostic = false; - static bool show_hit = false; - static bool show_ambig = false; - static bool show_profile = false; - static bool show_tokens = false; - static bool show_trace = false; - static bool show_tree = false; - static bool old = false; - static bool two_byte = false; - static int exit_code = 0; - static Encoding encoding = null; - static int string_instance = 0; - static string prefix = ""; - static bool quiet = false; - - static void Main(string[] args) - { - List is_fns = new List(); - List inputs = new List(); - for (int i = 0; i < args.Length; ++i) - { - if (args[i] == "-d") - { - show_diagnostic = true; - } - else if (args[i] == "-ambig") - { - show_ambig = true; - } - else if (args[i] == "-profile") - { - show_profile = true; - } - else if (args[i] == "-tokens") - { - show_tokens = true; - } - else if (args[i] == "-two-byte") - { - two_byte = true; - } - else if (args[i] == "-old") - { - old = true; - } - else if (args[i] == "-tree") - { - show_tree = true; - } - else if (args[i] == "-prefix") - { - prefix = args[++i] + " "; - } - else if (args[i] == "-input") - { - inputs.Add(args[++i]); - is_fns.Add(false); - } - else if (args[i] == "-tee") - { - tee = true; - } - else if (args[i] == "-encoding") - { - ++i; - encoding = Encoding.GetEncoding( - args[i], - new EncoderReplacementFallback("(unknown)"), - new DecoderReplacementFallback("(error)")); - if (encoding == null) - throw new Exception(@"Unknown encoding. Must be an Internet Assigned Numbers Authority (IANA) code page name. https://www.iana.org/assignments/character-sets/character-sets.xhtml"); - } - else if (args[i] == "-x") - { - for (; ; ) - { - var line = System.Console.In.ReadLine(); - line = line?.Trim(); - if (line == null || line == "") - { - break; - } - inputs.Add(line); - is_fns.Add(true); - } - } - else if (args[i] == "-q") - { - quiet = true; - } - else if (args[i] == "-trace") - { - show_trace = true; - } - else - { - inputs.Add(args[i]); - is_fns.Add(true); - } - } - if (inputs.Count() == 0) - { - ParseStdin(); - } - else - { - DateTime before = DateTime.Now; - for (int f = 0; f < inputs.Count(); ++f) - { - if (is_fns[f]) - ParseFilename(inputs[f], f); - else - ParseString(inputs[f], f); - } - DateTime after = DateTime.Now; - if (!quiet) System.Console.Error.WriteLine("Total Time: " + (after - before).TotalSeconds); - } - Environment.ExitCode = exit_code; - } - - static void ParseStdin() - { - ICharStream str = null; - str = CharStreams.fromStream(System.Console.OpenStandardInput()); - DoParse(str, "stdin", 0); - } - - static void ParseString(string input, int row_number) - { - ICharStream str = null; - str = CharStreams.fromString(input); - DoParse(str, "string" + string_instance++, row_number); - } - - static void ParseFilename(string input, int row_number) - { - ICharStream str = null; - if (two_byte) - str = new TwoByteCharStream(input); - else if (old) - { - FileStream fs = new FileStream(input, FileMode.Open); - str = new Antlr4.Runtime.AntlrInputStream(fs); - } - else if (encoding == null) - str = CharStreams.fromPath(input); - else - str = CharStreams.fromPath(input, encoding); - DoParse(str, input, row_number); - } - - static void DoParse(ICharStream str, string input_name, int row_number) - { - var lexer = new MySQLLexer(str); - lexer.serverVersion = 80200; - lexer.sqlModeFromString("ANSI_QUOTES"); - if (show_tokens) - { - StringBuilder new_s = new StringBuilder(); - for (int i = 0; ; ++i) - { - var ro_token = lexer.NextToken(); - var token = (CommonToken)ro_token; - token.TokenIndex = i; - new_s.AppendLine(token.ToString()); - if (token.Type == Antlr4.Runtime.TokenConstants.EOF) - break; - } - System.Console.Error.WriteLine(new_s.ToString()); - lexer.Reset(); - } - CommonTokenStream tokens = null; - if (show_hit) { - tokens = new ProfilingCommonTokenStream(lexer); - } - else { - tokens = new CommonTokenStream(lexer); - } - var parser = new MyParser(tokens); - parser.serverVersion = lexer.serverVersion; - parser.sqlModes = lexer.sqlModes; - var output = tee ? new StreamWriter(input_name + ".errors") : System.Console.Error; - var listener_lexer = new ErrorListener(quiet, tee, output); - var listener_parser = new ErrorListener(quiet, tee, output); - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - if (show_diagnostic) - { - parser.AddErrorListener(new MyDiagnosticErrorListener()); - } - if (show_profile || show_ambig) - { - parser.Profile = true; - } - if (show_trace) - { - parser.Trace = true; -// ParserATNSimulator.trace_atn_sim = true; - } - DateTime before = DateTime.Now; - var tree = parser.queries(); - DateTime after = DateTime.Now; - var result = ""; - if (listener_lexer.had_error || listener_parser.had_error) - { - result = "fail"; - exit_code = 1; - } - else - { - result = "success"; - } - if (show_tree) - { - if (tee) - { - System.IO.File.WriteAllText(input_name + ".tree", tree.ToStringTree(parser)); - } else { - System.Console.Error.WriteLine(tree.ToStringTree(parser)); - } - } - if (show_profile) - { - System.Console.Error.WriteLine(String.Join(",\n\r", parser.ParseInfo.getDecisionInfo().Select(d => d.ToString()))); - } - if (show_ambig) - { - var decs = parser.ParseInfo.getDecisionInfo().Where(d => - d.ambiguities.Any()).Select(d => d.ambiguities).ToList(); - foreach (var decision in decs) - { - foreach (var ai in decision) - { - var parser_decision = ai.decision; - var parser_alts = ai.ambigAlts; - var parser_startIndex = ai.startIndex; - var parser_stopIndex = ai.stopIndex; - var nfa_state = parser.Atn.states.Where(s => - { - if (s is BasicBlockStartState s2) return s2.decision == parser_decision; - else return false; - }).ToList(); - var p = parser.RuleNames.Select((value, index) => new { value, index }) - .Where(pair => (pair.value == "queries")) - .Select(pair => pair.index).First(); - var parser_startRuleIndex = p; - var parser_trees = parser.getAllPossibleParseTrees( - parser_decision, - parser_alts, - parser_startIndex, - parser_stopIndex, - parser_startRuleIndex); - foreach (var parser_tree in parser_trees) - { - System.Console.WriteLine(parser_tree.ToStringTree(parser)); - System.Console.WriteLine(); - } - } - } - } - if (!quiet) - { - System.Console.Error.WriteLine(prefix + "CSharp " + row_number + " " + input_name + " " + result + " " + (after - before).TotalSeconds); - } - if (tee) output.Close(); - } - - public static string ToStringTree(ITree tree, Parser recog) - { - StringBuilder sb = new StringBuilder(); - string[] ruleNames = recog != null ? recog.RuleNames : null; - IList ruleNamesList = ruleNames != null ? ruleNames.ToList() : null; - ToStringTree(sb, tree, 0, ruleNamesList); - return sb.ToString(); - } - - public static void ToStringTree(StringBuilder sb, ITree t, int indent, IList ruleNames) - { - string s = Antlr4.Runtime.Misc.Utils.EscapeWhitespace(GetNodeText(t, ruleNames), false); - if (t.ChildCount == 0) - { - for (int i = 0; i < indent; ++i) sb.Append(" "); - sb.AppendLine(s); - return; - } - s = Antlr4.Runtime.Misc.Utils.EscapeWhitespace(GetNodeText(t, ruleNames), false); - for (int i = 0; i < indent; ++i) sb.Append(' '); - sb.AppendLine(s); - for (int i = 0; i < t.ChildCount; i++) - { - ToStringTree(sb, t.GetChild(i), indent+1, ruleNames); - } - } - - public static string GetNodeText(ITree t, Parser recog) - { - string[] ruleNames = recog != null ? recog.RuleNames : null; - IList ruleNamesList = ruleNames != null ? ruleNames.ToList() : null; - return GetNodeText(t, ruleNamesList); - } - - public static string GetNodeText(ITree t, IList ruleNames) - { - if (ruleNames != null) - { - if (t is RuleContext) - { - int ruleIndex = ((RuleContext)t).RuleIndex; - string ruleName = ruleNames[ruleIndex]; - int altNumber = ((RuleContext)t).getAltNumber(); - if ( altNumber!= Antlr4.Runtime.Atn.ATN.INVALID_ALT_NUMBER ) { - return ruleName+":"+altNumber; - } - return ruleName; - } - else - { - if (t is IErrorNode) - { - return t.ToString(); - } - else - { - if (t is ITerminalNode) - { - IToken symbol = ((ITerminalNode)t).Symbol; - if (symbol != null) - { - string s = symbol.Text; - return s; - } - } - } - } - } - // no recog for rule names - object payload = t.Payload; - if (payload is IToken) - { - return ((IToken)payload).Text; - } - return t.Payload.ToString(); - } -} - diff --git a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart new file mode 100644 index 0000000000..7544f31ab0 --- /dev/null +++ b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart @@ -0,0 +1,387 @@ +import 'package:antlr4/antlr4.dart'; +import 'dart:io'; +import 'dart:core'; +import 'dart:convert'; +import 'dart:collection'; +import 'MySQLLexer.dart'; +import 'MySQLParser.dart'; + +/** SQL modes that control parsing behavior. */ +enum SqlMode { + noMode, + ansiQuotes, + highNotPrecedence, + pipesAsConcat, + ignoreSpace, + noBackslashEscapes, +} + +/** Base lexer class providing functions needed in actions. */ +abstract class MySQLLexerBase extends Lexer +{ + int serverVersion = 0; + Set sqlModes = {}; + bool supportMle = true; + Set charSets = {}; + bool inVersionComment = false; + Queue pendingTokens = Queue(); + + static const String longString = "2147483647"; + static const int longLength = 10; + static const String signedLongString = "-2147483648"; + static const String longLongString = "9223372036854775807"; + static const int longLongLength = 19; + static const String signedLongLongString = "-9223372036854775808"; + static const int signedLongLongLength = 19; + static const String unsignedLongLongString = "18446744073709551615"; + static const int unsignedLongLongLength = 20; + + bool justEmitedDot = false; + + MySQLLexerBase(CharStream input) : super(input) + { + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + + bool isSqlModeActive(SqlMode mode) { + return sqlModes.contains(mode); + } + + void sqlModeFromString(String modes) { + sqlModes.clear(); + List parts = modes.toUpperCase().split(','); + + for (String mode in parts) { + if (['ANSI', 'DB2', 'MAXDB', 'MSSQL', 'ORACLE', 'POSTGRESQL'] + .contains(mode)) { + sqlModes.add(SqlMode.ansiQuotes); + sqlModes.add(SqlMode.pipesAsConcat); + sqlModes.add(SqlMode.ignoreSpace); + } else if (mode == 'ANSI_QUOTES') { + sqlModes.add(SqlMode.ansiQuotes); + } else if (mode == 'PIPES_AS_CONCAT') { + sqlModes.add(SqlMode.pipesAsConcat); + } else if (mode == 'NO_BACKSLASH_ESCAPES') { + sqlModes.add(SqlMode.noBackslashEscapes); + } else if (mode == 'IGNORE_SPACE') { + sqlModes.add(SqlMode.ignoreSpace); + } else if (['HIGH_NOT_PRECEDENCE', 'MYSQL323', 'MYSQL40'] + .contains(mode)) { + sqlModes.add(SqlMode.highNotPrecedence); + } + } + } + + /** + * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. + */ + @override void reset([bool resetInput = false]) + { + inVersionComment = false; + super.reset(true); // reset() needs to follow the semantics in other targets! + } + + /** + * Implements the multi token feature required in our lexer. + * A lexer rule can emit more than a single token, if needed. + * + * @returns The next token in the token stream. + */ + @override Token nextToken() + { + // First respond with pending tokens to the next token request, if there are any. + if (! this.pendingTokens.isEmpty) { + var pending = this.pendingTokens.removeFirst(); + return pending; + } + + // Let the main lexer class run the next token recognition. + // This might create additional tokens again. + var next = super.nextToken(); + + if (! this.pendingTokens.isEmpty) { + var pending = this.pendingTokens.removeFirst(); + this.pendingTokens.add(next); + return pending; + } + + return next; + } + + /** + * Checks if the version number in the token text is less than or equal to the current server version. + * + * @param text The text from a matched token. + * @returns True if so the number matches, otherwise false. + */ + bool checkMySQLVersion(String text) { + if (text.length < 8) { + return false; + } + + int version = int.parse(text.substring(3)); + if (version <= serverVersion) { + inVersionComment = true; + return true; + } + + return false; + } + + /** + * Called when a keyword was consumed that represents an internal MySQL function and checks if that keyword is + * followed by an open parenthesis. If not then it is not considered a keyword but treated like a normal identifier. + * + * @param proposed The token type to use if the check succeeds. + * + * @returns If a function call is found then return the proposed token type, otherwise just IDENTIFIER. + */ + int determineFunction(int proposed) { + var input = this.inputStream.LA(1) ?? 0; + if (isSqlModeActive(SqlMode.ignoreSpace)) { + while ([' ', '\t', '\r', '\n'].contains(String.fromCharCode(input))) { + // Consume logic based on InputStream equivalent. + this.interpreter?.consume(this.inputStream); + this.channel = Lexer.HIDDEN; + this.type = MySQLLexer.TOKEN_WHITESPACE; + var c = this.inputStream.LA(1); + input = c ?? 0; + } + } + var r = input == '('.codeUnitAt(0) ? proposed : MySQLLexer.TOKEN_IDENTIFIER; + return r; + } + + /** + * Checks the given text and determines the smallest number type from it. Code has been taken from sql_lex.cc. + * + * @param text The text to parse (which must be a number). + * + * @returns The token type for that text. + */ + int determineNumericType(String text) + { + // The original code checks for leading +/- but actually that can never happen, neither in the + // server parser (as a digit is used to trigger processing in the lexer) nor in our parser + // as our rules are defined without signs. But we do it anyway for maximum compatibility. + var length = text.length - 1; + if (length < MySQLLexerBase.longLength) { // quick normal case + return MySQLLexer.TOKEN_INT_NUMBER; + } + + var negative = false; + var index = 0; + if (text[index] == '+') { // Remove sign and pre-zeros + ++index; + --length; + } else if (text[index] == '-') { + ++index; + --length; + negative = true; + } + + while (text[index] == '0' && length > 0) { + ++index; + --length; + } + + if (length < MySQLLexerBase.longLength) { + return MySQLLexer.TOKEN_INT_NUMBER; + } + + int smaller; + int bigger; + String cmp; + if (negative) { + if (length == MySQLLexerBase.longLength) { + cmp = MySQLLexerBase.signedLongString.substring(1); + smaller = MySQLLexer.TOKEN_INT_NUMBER; // If <= signed_long_str + bigger = MySQLLexer.TOKEN_LONG_NUMBER; // If >= signed_long_str + } else if (length < MySQLLexerBase.signedLongLongLength) { + return MySQLLexer.TOKEN_LONG_NUMBER; + } else if (length > MySQLLexerBase.signedLongLongLength) { + return MySQLLexer.TOKEN_DECIMAL_NUMBER; + } else { + cmp = MySQLLexerBase.signedLongLongString.substring(1); + smaller = MySQLLexer.TOKEN_LONG_NUMBER; // If <= signed_longlong_str + bigger = MySQLLexer.TOKEN_DECIMAL_NUMBER; + } + } else { + if (length == MySQLLexerBase.longLength) { + cmp = MySQLLexerBase.longString; + smaller = MySQLLexer.TOKEN_INT_NUMBER; + bigger = MySQLLexer.TOKEN_LONG_NUMBER; + } else if (length < MySQLLexerBase.longLongLength) { + return MySQLLexer.TOKEN_LONG_NUMBER; + } else if (length > MySQLLexerBase.longLongLength) { + if (length > MySQLLexerBase.unsignedLongLongLength) { + return MySQLLexer.TOKEN_DECIMAL_NUMBER; + } + cmp = MySQLLexerBase.unsignedLongLongString; + smaller = MySQLLexer.TOKEN_ULONGLONG_NUMBER; + bigger = MySQLLexer.TOKEN_DECIMAL_NUMBER; + } else { + cmp = MySQLLexerBase.longLongString; + smaller = MySQLLexer.TOKEN_LONG_NUMBER; + bigger = MySQLLexer.TOKEN_ULONGLONG_NUMBER; + } + } + + var otherIndex = 0; + while (index < text.length && cmp[otherIndex++] == text[index++]) { + // + } + + var i = text[index - 1].compareTo(cmp[otherIndex - 1]); + return i <= 0 ? smaller : bigger; + } + + /** + * Checks if the given text corresponds to a charset defined in the server (text is preceded by an underscore). + * + * @param text The text to check. + * + * @returns UNDERSCORE_CHARSET if so, otherwise IDENTIFIER. + */ + int checkCharset(String text) + { + var z = this.charSets.contains(text); + var r = z ? MySQLLexer.TOKEN_UNDERSCORE_CHARSET : MySQLLexer.TOKEN_IDENTIFIER; + return r; + } + + /** + * Creates a DOT token in the token stream. + */ + void emitDot() + { + var ctf = this.tokenFactory; + Token t = ctf.create( + MySQLLexer.TOKEN_DOT_SYMBOL, + this.text, + Pair(this, this.inputStream), + this.channel, + this.tokenStartCharIndex, + this.tokenStartCharIndex, + this.line, + this.charPositionInLine); + this.pendingTokens.add(t); + ++this.charPositionInLine; + ++this.tokenStartCharIndex; + this.justEmitedDot = true; + } + + // Version-related methods + bool isServerVersionLt80024() => serverVersion < 80024; + bool isServerVersionGe80024() => serverVersion >= 80024; + bool isServerVersionGe80011() => serverVersion >= 80011; + bool isServerVersionGe80013() => serverVersion >= 80013; + bool isServerVersionLt80014() => serverVersion < 80014; + bool isServerVersionGe80014() => serverVersion >= 80014; + bool isServerVersionGe80017() => serverVersion >= 80017; + bool isServerVersionGe80018() => serverVersion >= 80018; + + bool isMasterCompressionAlgorithm() => serverVersion >= 80018 && isServerVersionLt80024(); + + bool isServerVersionLt80031() => serverVersion < 80031; + + // Functions for specific token types + void doLogicalOr() + { + this.type = isSqlModeActive(SqlMode.pipesAsConcat) ? MySQLLexer.TOKEN_CONCAT_PIPES_SYMBOL : MySQLLexer.TOKEN_LOGICAL_OR_OPERATOR; + } + + void doIntNumber() + { + this.type = determineNumericType(this.text); + } + + void doAdddate() => this.type = determineFunction(MySQLLexer.TOKEN_ADDDATE_SYMBOL); + void doBitAnd() => this.type = determineFunction(MySQLLexer.TOKEN_BIT_AND_SYMBOL); + void doBitOr() => this.type = determineFunction(MySQLLexer.TOKEN_BIT_OR_SYMBOL); + void doBitXor() => this.type = determineFunction(MySQLLexer.TOKEN_BIT_XOR_SYMBOL); + void doCast() => this.type = determineFunction(MySQLLexer.TOKEN_CAST_SYMBOL); + void doCount() => this.type = determineFunction(MySQLLexer.TOKEN_COUNT_SYMBOL); + void doCurdate() => this.type = determineFunction(MySQLLexer.TOKEN_CURDATE_SYMBOL); + void doCurrentDate() => this.type = determineFunction(MySQLLexer.TOKEN_CURDATE_SYMBOL); + void doCurrentTime() => this.type = determineFunction(MySQLLexer.TOKEN_CURTIME_SYMBOL); + void doCurtime() => this.type = determineFunction(MySQLLexer.TOKEN_CURTIME_SYMBOL); + void doDateAdd() => this.type = determineFunction(MySQLLexer.TOKEN_DATE_ADD_SYMBOL); + void doDateSub() => this.type = determineFunction(MySQLLexer.TOKEN_DATE_SUB_SYMBOL); + void doExtract() => this.type = determineFunction(MySQLLexer.TOKEN_EXTRACT_SYMBOL); + void doGroupConcat() => this.type = determineFunction(MySQLLexer.TOKEN_GROUP_CONCAT_SYMBOL); + void doMax() => this.type = determineFunction(MySQLLexer.TOKEN_MAX_SYMBOL); + void doMid() => this.type = determineFunction(MySQLLexer.TOKEN_SUBSTRING_SYMBOL); + void doMin() => this.type = determineFunction(MySQLLexer.TOKEN_MIN_SYMBOL); + void doNot() => this.type = isSqlModeActive(SqlMode.highNotPrecedence) ? MySQLLexer.TOKEN_NOT2_SYMBOL : MySQLLexer.TOKEN_NOT_SYMBOL; + void doNow() => this.type = determineFunction(MySQLLexer.TOKEN_NOW_SYMBOL); + void doPosition() => this.type = determineFunction(MySQLLexer.TOKEN_POSITION_SYMBOL); + void doSessionUser() => this.type = determineFunction(MySQLLexer.TOKEN_USER_SYMBOL); + void doStddevSamp() => this.type = determineFunction(MySQLLexer.TOKEN_STDDEV_SAMP_SYMBOL); + void doStddev() => this.type = determineFunction(MySQLLexer.TOKEN_STD_SYMBOL); + void doStddevPop() => this.type = determineFunction(MySQLLexer.TOKEN_STD_SYMBOL); + void doStd() => this.type = determineFunction(MySQLLexer.TOKEN_STD_SYMBOL); + void doSubdate() => this.type = determineFunction(MySQLLexer.TOKEN_SUBDATE_SYMBOL); + void doSubstr() => this.type = determineFunction(MySQLLexer.TOKEN_SUBSTRING_SYMBOL); + void doSubstring() => this.type = determineFunction(MySQLLexer.TOKEN_SUBSTRING_SYMBOL); + void doSum() => this.type = determineFunction(MySQLLexer.TOKEN_SUM_SYMBOL); + void doSysdate() => this.type = determineFunction(MySQLLexer.TOKEN_SYSDATE_SYMBOL); + void doSystemUser() => this.type = determineFunction(MySQLLexer.TOKEN_USER_SYMBOL); + void doTrim() => this.type = determineFunction(MySQLLexer.TOKEN_TRIM_SYMBOL); + void doVariance() => this.type = determineFunction(MySQLLexer.TOKEN_VARIANCE_SYMBOL); + void doVarPop() => this.type = determineFunction(MySQLLexer.TOKEN_VARIANCE_SYMBOL); + void doVarSamp() => this.type = determineFunction(MySQLLexer.TOKEN_VAR_SAMP_SYMBOL); + void doUnderscoreCharset() => this.type = checkCharset(this.text); + + bool isVersionComment() => checkMySQLVersion(this.text); + + bool isBackTickQuotedId() + { + return !this.isSqlModeActive(SqlMode.noBackslashEscapes); + } + + bool isDoubleQuotedText() + { + return !this.isSqlModeActive(SqlMode.noBackslashEscapes); + } + + bool isSingleQuotedText() + { + return !this.isSqlModeActive(SqlMode.noBackslashEscapes); + } + +/* + @override Token emit() + { + var ctf = this.tokenFactory; + Token t = ctf.create( + this.type, + (this.text!=null?(this.justEmitedDot?this.text.substring(1):this.text):null), + Pair(this, this.inputStream), + this.channel, + this.tokenStartCharIndex + (this.justEmitedDot?1:0), + CharIndex - 1, + this.line, + this.charPositionInLine); + this.justEmitedDot = false; + super.emit(t); + return t; + } +*/ + + void startInVersionComment() + { + inVersionComment = true; + } + + void endInVersionComment() + { + inVersionComment = false; + } + + bool isInVersionComment() + { + return inVersionComment; + } +} diff --git a/sql/mysql/Oracle/Dart/MySQLParserBase.dart b/sql/mysql/Oracle/Dart/MySQLParserBase.dart new file mode 100644 index 0000000000..47f1c61b8f --- /dev/null +++ b/sql/mysql/Oracle/Dart/MySQLParserBase.dart @@ -0,0 +1,82 @@ +import 'package:antlr4/antlr4.dart'; +import 'dart:io'; +import 'dart:core'; +import 'dart:convert'; +import 'dart:collection'; +import 'MySQLLexer.dart'; +import 'MySQLParser.dart'; + +/** SQL modes that control parsing behavior. */ +enum SqlMode { + noMode, + ansiQuotes, + highNotPrecedence, + pipesAsConcat, + ignoreSpace, + noBackslashEscapes, +} + +/** Base parser class for MySQL parsing. */ +abstract class MySQLParserBase extends Parser { + + // To parameterize the parsing process. + int serverVersion = 0; + Set sqlModes = {}; + bool supportMle = true; + + MySQLParserBase(TokenStream input) : super(input) + { + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + + /// Determines if the given SQL mode is currently active. + bool isSqlModeActive(SqlMode mode) { + return sqlModes.contains(mode); + } + + /// Checks if the parser is in pure identifier mode. + bool isPureIdentifier() { + return isSqlModeActive(SqlMode.ansiQuotes); + } + + /// Checks if the parser is in text string literal mode. + bool isTextStringLiteral() { + return !isSqlModeActive(SqlMode.ansiQuotes); + } + + /// Checks if the parser is handling a stored routine body. + bool isStoredRoutineBody() { + return serverVersion >= 80032 && supportMle; + } + + /// Checks if the parser is handling a SELECT statement with INTO clause. + bool isSelectStatementWithInto() { + return serverVersion >= 80024 && serverVersion < 80031; + } + + void sqlModeFromString(String modes) { + sqlModes.clear(); + List parts = modes.toUpperCase().split(','); + + for (String mode in parts) { + if (['ANSI', 'DB2', 'MAXDB', 'MSSQL', 'ORACLE', 'POSTGRESQL'] + .contains(mode)) { + sqlModes.add(SqlMode.ansiQuotes); + sqlModes.add(SqlMode.pipesAsConcat); + sqlModes.add(SqlMode.ignoreSpace); + } else if (mode == 'ANSI_QUOTES') { + sqlModes.add(SqlMode.ansiQuotes); + } else if (mode == 'PIPES_AS_CONCAT') { + sqlModes.add(SqlMode.pipesAsConcat); + } else if (mode == 'NO_BACKSLASH_ESCAPES') { + sqlModes.add(SqlMode.noBackslashEscapes); + } else if (mode == 'IGNORE_SPACE') { + sqlModes.add(SqlMode.ignoreSpace); + } else if (['HIGH_NOT_PRECEDENCE', 'MYSQL323', 'MYSQL40'] + .contains(mode)) { + sqlModes.add(SqlMode.highNotPrecedence); + } + } + } +} diff --git a/sql/mysql/Oracle/Java/MySQLLexerBase.java b/sql/mysql/Oracle/Java/MySQLLexerBase.java index c020d4a162..fc6f7ae4a3 100644 --- a/sql/mysql/Oracle/Java/MySQLLexerBase.java +++ b/sql/mysql/Oracle/Java/MySQLLexerBase.java @@ -25,7 +25,7 @@ public MySQLLexerBase(CharStream input) { public Set charSets = new HashSet<>(); // Used to check repertoires. protected boolean inVersionComment = false; - private Deque pendingTokens = new ArrayDeque<>(); + private Queue pendingTokens = new LinkedList<>(); static String longString = "2147483647"; static int longLength = 10; @@ -100,16 +100,16 @@ public void reset() { @Override public Token nextToken() { - Token pending = pendingTokens.pollLast(); + Token pending = pendingTokens.poll(); if (pending != null) { return pending; } Token next = super.nextToken(); - pending = pendingTokens.pollLast(); + pending = pendingTokens.poll(); if (pending != null) { - pendingTokens.push(next); + pendingTokens.add(next); return pending; } @@ -218,7 +218,7 @@ protected int checkCharset(String text) { } protected void emitDot() { - pendingTokens.push(this._factory.create(this._tokenFactorySourcePair, MySQLLexer.DOT_SYMBOL, + pendingTokens.add(this._factory.create(this._tokenFactorySourcePair, MySQLLexer.DOT_SYMBOL, this.getText(), this._channel, this._tokenStartCharIndex, this._tokenStartCharIndex, this.getLine(), this.getCharPositionInLine())); ++this._tokenStartCharPositionInLine; this.justEmitedDot = true; diff --git a/sql/mysql/Oracle/desc.xml b/sql/mysql/Oracle/desc.xml index 5819861f41..78656ebb33 100644 --- a/sql/mysql/Oracle/desc.xml +++ b/sql/mysql/Oracle/desc.xml @@ -1,5 +1,5 @@ - Antlr4ng;CSharp;Java;Python3;TypeScript + Antlr4ng;CSharp;Dart;Java;Python3;TypeScript examples/**/*.sql From fce1f94591623b84d324693675d424de1c45d357 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Fri, 29 Nov 2024 06:58:27 -0500 Subject: [PATCH 02/29] For Python3 port, move custom initialization to constructor in base classes where it belongs. --- sql/mysql/Oracle/Python3/MySQLLexerBase.py | 6 +- sql/mysql/Oracle/Python3/MySQLParserBase.py | 25 ++- sql/mysql/Oracle/Python3/Test.py | 201 -------------------- 3 files changed, 26 insertions(+), 206 deletions(-) delete mode 100644 sql/mysql/Oracle/Python3/Test.py diff --git a/sql/mysql/Oracle/Python3/MySQLLexerBase.py b/sql/mysql/Oracle/Python3/MySQLLexerBase.py index 967cc32e19..5e9a2796b9 100644 --- a/sql/mysql/Oracle/Python3/MySQLLexerBase.py +++ b/sql/mysql/Oracle/Python3/MySQLLexerBase.py @@ -3,9 +3,9 @@ from collections import deque import sys if sys.version_info[1] > 5: - from typing import TextIO + from typing import TextIO else: - from typing.io import TextIO + from typing.io import TextIO from SqlMode import SqlMode from MySQLParser import MySQLParser @@ -29,6 +29,8 @@ def __init__(self, input=None, output:TextIO = sys.stdout): self.inVersionComment = False self.pendingTokens = deque() self.justEmitedDot = False + self.serverVersion = 80200; + self.sqlModeFromString("ANSI_QUOTES"); def isSqlModeActive(self, mode): return mode in self.sqlModes diff --git a/sql/mysql/Oracle/Python3/MySQLParserBase.py b/sql/mysql/Oracle/Python3/MySQLParserBase.py index 12c789fb1d..b6051d4d87 100644 --- a/sql/mysql/Oracle/Python3/MySQLParserBase.py +++ b/sql/mysql/Oracle/Python3/MySQLParserBase.py @@ -2,9 +2,9 @@ import re import sys if sys.version_info[1] > 5: - from typing import TextIO + from typing import TextIO else: - from typing.io import TextIO + from typing.io import TextIO from SqlMode import SqlMode class MySQLParserBase(Parser): @@ -13,6 +13,8 @@ def __init__(self, input:TokenStream, output:TextIO = sys.stdout): self.serverVersion = 0 self.sqlModes = set() self.supportMle = True + self.serverVersion = 80200; + self.sqlModeFromString("ANSI_QUOTES"); def isSqlModeActive(self, mode): """ @@ -42,4 +44,21 @@ def isStoredRoutineBody(self): def isSelectStatementWithInto(self): return self.serverVersion >= 80024 and self.serverVersion < 80031; - \ No newline at end of file + + def sqlModeFromString(self, modes): + self.sqlModes.clear() + parts = modes.upper().split(",") + for mode in parts: + if mode in {"ANSI", "DB2", "MAXDB", "MSSQL", "ORACLE", "POSTGRESQL"}: + self.sqlModes.update({SqlMode.AnsiQuotes, SqlMode.PipesAsConcat, SqlMode.IgnoreSpace}) + elif mode == "ANSI_QUOTES": + self.sqlModes.add(SqlMode.AnsiQuotes) + elif mode == "PIPES_AS_CONCAT": + self.sqlModes.add(SqlMode.PipesAsConcat) + elif mode == "NO_BACKSLASH_ESCAPES": + self.sqlModes.add(SqlMode.NoBackslashEscapes) + elif mode == "IGNORE_SPACE": + self.sqlModes.add(SqlMode.IgnoreSpace) + elif mode in {"HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40"}: + self.sqlModes.add(SqlMode.HighNotPrecedence) + diff --git a/sql/mysql/Oracle/Python3/Test.py b/sql/mysql/Oracle/Python3/Test.py deleted file mode 100644 index 1351d3e57c..0000000000 --- a/sql/mysql/Oracle/Python3/Test.py +++ /dev/null @@ -1,201 +0,0 @@ -# Generated from trgen 0.23.7 - -import sys -from antlr4 import * -from antlr4.error.ErrorListener import ErrorListener -from readchar import readchar -from MySQLLexer import MySQLLexer; -from MySQLParser import MySQLParser; -from datetime import datetime - -def getChar(): - xx = readchar() - if (xx == 0): - return ''; - return xx - -class MyErrorListener(ErrorListener): - - def __init__(self, q, t, o): - super().__init__() - self.had_error = False - self.quiet = q - self.tee = t; - self.output = o - - def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): - self.had_error = True - if ( self.tee ): - self.output.write(f"line {line}:{column} {msg}\n"); - if (not self.quiet): - print(f"line {line}:{column} {msg}", file=sys.stderr); - -tee = False -show_tokens = False -show_tree = False -show_trace = False -encoding = "utf-8" -error_code = 0 -string_instance = 0 -prefix = "" -quiet = False - -def main(argv): - global tee - global show_tokens - global show_tree - global show_trace - global encoding - global prefix - global quiet - global error_code - - inputs = [] - is_fns = [] - prefix = "" - i = 1 - while i < len(argv): - arg = argv[i] - if arg == "-tokens": - show_tokens = True - elif arg == "-tree": - show_tree = True - elif arg == "-prefix": - i = i + 1 - prefix = argv[i] + " " - elif arg == "-input": - i = i + 1 - inputs.append(argv[i]) - is_fns.append(False) - elif arg == "-encoding": - i = i + 1 - encoding = argv[i] - elif arg == "-tee": - tee = True - elif arg == "-x": - while f := sys.stdin.readline(): - f = f.strip() - inputs.append(f) - is_fns.append(True) - elif arg == "-q": - quiet = True - elif arg == "-trace": - show_trace = True - else: - inputs.append(argv[i]) - is_fns.append(True) - i = i + 1 - if len(inputs) == 0: - ParseStdin() - else: - start_time = datetime.now() - for f in range(0, len(inputs)): - if is_fns[f]: - ParseFilename(inputs[f], f) - else: - ParseString(inputs[f], f) - end_time = datetime.now() - diff = end_time - start_time - diff_time = diff.total_seconds() - if (not quiet): - print(f'Total Time: {diff_time}', file=sys.stderr); - sys.exit(error_code) - -def ParseStdin(): - sb = "" - ch = getChar() - while (ch != ''): - sb = sb + ch - ch = getChar() - input = sb - str = InputStream(input); - DoParse(str, 'stdin', 0) - -def ParseString(input, row_number): - global string_instance - str = InputStream(input) - DoParse(str, 'string', row_number) - string_instance = string_instance + 1 - -def ParseFilename(input, row_number): - global encoding - str = FileStream(input, encoding) - DoParse(str, input, row_number) - -def DoParse(str, input_name, row_number): - global tee - global show_tokens - global show_tree - global show_trace - global encoding - global prefix - global quiet - global error_code - - lexer = MySQLLexer(str) - lexer.serverVersion = 80200; - lexer.sqlModeFromString("ANSI_QUOTES"); - lexer.removeErrorListeners() - if (tee): - output = open(input_name + ".errors", "w") - else: - output = sys.stderr - listener_lexer = MyErrorListener(quiet, tee, output) - lexer.addErrorListener(listener_lexer) - # lexer.strictMode = false - tokens = CommonTokenStream(lexer) - parser = MySQLParser(tokens) - parser.serverVersion = lexer.serverVersion; - parser.sqlModes = lexer.sqlModes; - parser.removeErrorListeners() - listener_parser = MyErrorListener(quiet, tee, output) - parser.addErrorListener(listener_parser) - if (show_tokens): - i = 0 - while True: - ro_token = lexer.nextToken() - token = ro_token - # token.TokenIndex = i - i = i + 1 - print(token, file=sys.stderr) - if (token.type == -1): - break - lexer.reset() - if (show_trace) : - parser.setTrace(False) - ParserATNSimulator.trace_atn_sim = True - PredictionContext._trace_atn_sim = True - start_time = datetime.now() - tree = parser.queries() - end_time = datetime.now() - diff = end_time - start_time - diff_time = diff.total_seconds() - result = '' - if listener_parser.had_error or listener_lexer.had_error: - result = 'fail' - error_code = 1 - else: - result = 'success' - if (show_tree): - if (tee): - f = open(input_name + '.tree', 'w', encoding='utf-8') - f.write(tree.toStringTree(recog=parser)) - f.close() - else: - print(tree.toStringTree(recog=parser), file=sys.stderr) - if (not quiet): - sys.stderr.write(prefix) - sys.stderr.write('Python3 ') - sys.stderr.write(f'{row_number}') - sys.stderr.write(' ') - sys.stderr.write(input_name) - sys.stderr.write(' ') - sys.stderr.write(result) - sys.stderr.write(' ') - sys.stderr.write(f'{diff_time}') - sys.stderr.write('\n') - if (tee): - output.close() - -if __name__ == '__main__': - main(sys.argv) From 5ba847f0b38fcdd23676e63ef91dffe3f37c1a76 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Fri, 29 Nov 2024 07:58:35 -0500 Subject: [PATCH 03/29] Moved SqlMode to separate file for Antlr4ng. Fixed constructor for lexer and parser in TypeScript port. --- sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts | 40 +------ sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts | 2 +- sql/mysql/Oracle/Antlr4ng/SqlMode.ts | 11 ++ sql/mysql/Oracle/Antlr4ng/transformGrammar.py | 3 +- sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts | 109 +++++++++--------- .../Oracle/TypeScript/MySQLParserBase.ts | 43 ++++++- 6 files changed, 110 insertions(+), 98 deletions(-) create mode 100644 sql/mysql/Oracle/Antlr4ng/SqlMode.ts diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts index 3a418acefb..307c113912 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts @@ -7,16 +7,7 @@ import { CharStream, Lexer, Token } from "antlr4ng"; import { MySQLLexer } from "./MySQLLexer.js"; - -/** SQL modes that control parsing behavior. */ -export enum SqlMode { - NoMode, - AnsiQuotes, - HighNotPrecedence, - PipesAsConcat, - IgnoreSpace, - NoBackslashEscapes, -} +import SqlMode from "./SqlMode.js"; /** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ export abstract class MySQLLexerBase extends Lexer { @@ -44,7 +35,7 @@ export abstract class MySQLLexerBase extends Lexer { constructor(input: CharStream) { super(input); this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** @@ -58,33 +49,6 @@ export abstract class MySQLLexerBase extends Lexer { return this.sqlModes.has(mode); } - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public sqlModeFromString(modes: string): void { - this.sqlModes = new Set(); - - const parts = modes.toUpperCase().split(","); - parts.forEach((mode: string) => { - if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || - mode === "POSTGRESQL") { - this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); - } else if (mode === "ANSI_QUOTES") { - this.sqlModes.add(SqlMode.AnsiQuotes); - } else if (mode === "PIPES_AS_CONCAT") { - this.sqlModes.add(SqlMode.PipesAsConcat); - } else if (mode === "NO_BACKSLASH_ESCAPES") { - this.sqlModes.add(SqlMode.NoBackslashEscapes); - } else if (mode === "IGNORE_SPACE") { - this.sqlModes.add(SqlMode.IgnoreSpace); - } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { - this.sqlModes.add(SqlMode.HighNotPrecedence); - } - }); - } - /** * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. */ diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts index cbe88f5bf5..74ad3d8be7 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts @@ -3,7 +3,7 @@ */ import { Parser, TokenStream } from "antlr4ng"; -import { SqlMode } from "./MySQLLexerBase.js"; +import SqlMode from "./SqlMode.js"; export abstract class MySQLParserBase extends Parser { diff --git a/sql/mysql/Oracle/Antlr4ng/SqlMode.ts b/sql/mysql/Oracle/Antlr4ng/SqlMode.ts new file mode 100644 index 0000000000..acebb55637 --- /dev/null +++ b/sql/mysql/Oracle/Antlr4ng/SqlMode.ts @@ -0,0 +1,11 @@ +/** SQL modes that control parsing behavior. */ +enum SqlMode { + NoMode, + AnsiQuotes, + HighNotPrecedence, + PipesAsConcat, + IgnoreSpace, + NoBackslashEscapes, +} + +export default SqlMode; diff --git a/sql/mysql/Oracle/Antlr4ng/transformGrammar.py b/sql/mysql/Oracle/Antlr4ng/transformGrammar.py index 90b110076b..a250ae80db 100644 --- a/sql/mysql/Oracle/Antlr4ng/transformGrammar.py +++ b/sql/mysql/Oracle/Antlr4ng/transformGrammar.py @@ -26,7 +26,7 @@ def fix(file_path): /* eslint-disable @typescript-eslint/no-unused-vars, no-useless-escape */ -import { MySQLLexerBase, SqlMode } from "./MySQLLexerBase.js"; +import { MySQLLexerBase } from "./MySQLLexerBase.js"; } ''' ) @@ -37,7 +37,6 @@ def fix(file_path): /* eslint-disable no-useless-escape, no-lone-blocks */ import { MySQLParserBase } from "./MySQLParserBase.js"; -import { SqlMode } from "./MySQLLexerBase.js"; } ''' ) diff --git a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts index 12cec1412d..f9b3c80949 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts @@ -5,8 +5,7 @@ /* eslint-disable no-underscore-dangle */ /* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ -import { Lexer } from "antlr4"; -import { Token } from "antlr4"; +import { CharStream, Lexer, Token } from "antlr4"; import { CommonToken } from "antlr4"; import MySQLLexer from "./MySQLLexer"; import SqlMode from "./SqlMode"; @@ -34,6 +33,12 @@ export default abstract class MySQLLexerBase extends Lexer { static #unsignedLongLongString = "18446744073709551615"; static #unsignedLongLongLength = 20; + constructor(input: CharStream) { + super(input); + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + /** * Determines if the given SQL mode is currently active in the lexer. * @@ -250,8 +255,8 @@ export default abstract class MySQLLexerBase extends Lexer { * Creates a DOT token in the token stream. */ protected emitDot(): void { - let t = new CommonToken([this, this._input], - MySQLLexer.DOT_SYMBOL, 0, this._tokenStartCharIndex, this._tokenStartCharIndex); + let t = new CommonToken([this, this._input], + MySQLLexer.DOT_SYMBOL, 0, this._tokenStartCharIndex, this._tokenStartCharIndex); this.pendingTokens.push(t); ++this.column; ++this._tokenStartCharIndex; @@ -304,234 +309,234 @@ export default abstract class MySQLLexerBase extends Lexer { public doLogicalOr(): void { - this._type = this.isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; + this._type = this.isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; } public doIntNumber(): void { - this._type = this.determineNumericType(this.text); + this._type = this.determineNumericType(this.text); } public doAdddate(): void { - this._type = this.determineFunction(MySQLLexer.ADDDATE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.ADDDATE_SYMBOL); } public doBitAnd(): void { - this._type = this.determineFunction(MySQLLexer.BIT_AND_SYMBOL); + this._type = this.determineFunction(MySQLLexer.BIT_AND_SYMBOL); } public doBitOr(): void { - this._type = this.determineFunction(MySQLLexer.BIT_OR_SYMBOL); + this._type = this.determineFunction(MySQLLexer.BIT_OR_SYMBOL); } public doBitXor(): void { - this._type = this.determineFunction(MySQLLexer.BIT_XOR_SYMBOL); + this._type = this.determineFunction(MySQLLexer.BIT_XOR_SYMBOL); } public doCast(): void { - this._type = this.determineFunction(MySQLLexer.CAST_SYMBOL); + this._type = this.determineFunction(MySQLLexer.CAST_SYMBOL); } public doCount(): void { - this._type = this.determineFunction(MySQLLexer.COUNT_SYMBOL); + this._type = this.determineFunction(MySQLLexer.COUNT_SYMBOL); } public doCurdate(): void { - this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); } public doCurrentDate(): void { - this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); } public doCurrentTime(): void { - this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); } public doCurtime(): void { - this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); } public doDateAdd(): void { - this._type = this.determineFunction(MySQLLexer.DATE_ADD_SYMBOL); + this._type = this.determineFunction(MySQLLexer.DATE_ADD_SYMBOL); } public doDateSub(): void { - this._type = this.determineFunction(MySQLLexer.DATE_SUB_SYMBOL); + this._type = this.determineFunction(MySQLLexer.DATE_SUB_SYMBOL); } public doExtract(): void { - this._type = this.determineFunction(MySQLLexer.EXTRACT_SYMBOL); + this._type = this.determineFunction(MySQLLexer.EXTRACT_SYMBOL); } public doGroupConcat(): void { - this._type = this.determineFunction(MySQLLexer.GROUP_CONCAT_SYMBOL); + this._type = this.determineFunction(MySQLLexer.GROUP_CONCAT_SYMBOL); } public doMax(): void { - this._type = this.determineFunction(MySQLLexer.MAX_SYMBOL); + this._type = this.determineFunction(MySQLLexer.MAX_SYMBOL); } public doMid(): void { - this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doMin(): void { - this._type = this.determineFunction(MySQLLexer.MIN_SYMBOL); + this._type = this.determineFunction(MySQLLexer.MIN_SYMBOL); } public doNot(): void { - this._type = this.isSqlModeActive(SqlMode.HighNotPrecedence) ? MySQLLexer.NOT2_SYMBOL: MySQLLexer.NOT_SYMBOL; + this._type = this.isSqlModeActive(SqlMode.HighNotPrecedence) ? MySQLLexer.NOT2_SYMBOL: MySQLLexer.NOT_SYMBOL; } public doNow(): void { - this._type = this.determineFunction(MySQLLexer.NOW_SYMBOL); + this._type = this.determineFunction(MySQLLexer.NOW_SYMBOL); } public doPosition(): void { - this._type = this.determineFunction(MySQLLexer.POSITION_SYMBOL); + this._type = this.determineFunction(MySQLLexer.POSITION_SYMBOL); } public doSessionUser(): void { - this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); + this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); } public doStddevSamp(): void { - this._type = this.determineFunction(MySQLLexer.STDDEV_SAMP_SYMBOL); + this._type = this.determineFunction(MySQLLexer.STDDEV_SAMP_SYMBOL); } public doStddev(): void { - this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doStddevPop(): void { - this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doStd(): void { - this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); } public doSubdate(): void { - this._type = this.determineFunction(MySQLLexer.SUBDATE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SUBDATE_SYMBOL); } public doSubstr(): void { - this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doSubstring(): void { - this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); } public doSum(): void { - this._type = this.determineFunction(MySQLLexer.SUM_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SUM_SYMBOL); } public doSysdate(): void { - this._type = this.determineFunction(MySQLLexer.SYSDATE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.SYSDATE_SYMBOL); } public doSystemUser(): void { - this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); + this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); } public doTrim(): void { - this._type = this.determineFunction(MySQLLexer.TRIM_SYMBOL); + this._type = this.determineFunction(MySQLLexer.TRIM_SYMBOL); } public doVariance(): void { - this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); } public doVarPop(): void { - this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); } public doVarSamp(): void { - this._type = this.determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); + this._type = this.determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); } public doUnderscoreCharset(): void { - this._type = this.checkCharset(this.text); + this._type = this.checkCharset(this.text); } public isVersionComment(): boolean { - return this.checkMySQLVersion(this.text); + return this.checkMySQLVersion(this.text); } public isBackTickQuotedId(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public isDoubleQuotedText(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public isSingleQuotedText(): boolean { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } public override emit(): Token { - let t = super.emit(); - if (t.type == MySQLLexer.WHITESPACE) - t.channel = Token.HIDDEN_CHANNEL; + let t = super.emit(); + if (t.type == MySQLLexer.WHITESPACE) + t.channel = Token.HIDDEN_CHANNEL; return t; } public startInVersionComment(): void { - this.inVersionComment = true; + this.inVersionComment = true; } public endInVersionComment(): void { - this.inVersionComment = false; + this.inVersionComment = false; } public isInVersionComment(): boolean { - return this.inVersionComment; + return this.inVersionComment; } } diff --git a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts index 231280f665..0f6e1f0480 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts @@ -2,7 +2,7 @@ * Copyright © 2024, Oracle and/or its affiliates */ -import { Parser } from "antlr4"; +import { Parser, TokenStream } from "antlr4"; import MySQLParser from "./MySQLParser"; import SqlMode from "./SqlMode"; @@ -15,6 +15,12 @@ export default abstract class MySQLParserBase extends Parser { /** Enable Multi Language Extension support. */ public supportMle = true; + constructor(input: TokenStream) { + super(input); + this.serverVersion = 80200; + this.sqlModeFromString("ANSI_QUOTES"); + } + /** * Determines if the given SQL mode is currently active in the lexer. * @@ -28,21 +34,48 @@ export default abstract class MySQLParserBase extends Parser { public isPureIdentifier(): boolean { - return this.isSqlModeActive(SqlMode.AnsiQuotes); + return this.isSqlModeActive(SqlMode.AnsiQuotes); } public isTextStringLiteral(): boolean { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); + return !this.isSqlModeActive(SqlMode.AnsiQuotes); } public isStoredRoutineBody(): boolean { - return this.serverVersion >= 80032 && this.supportMle; + return this.serverVersion >= 80032 && this.supportMle; } public isSelectStatementWithInto(): boolean { - return this.serverVersion >= 80024 && this.serverVersion < 80031; + return this.serverVersion >= 80024 && this.serverVersion < 80031; + } + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public sqlModeFromString(modes: string): void { + this.sqlModes = new Set(); + + const parts = modes.toUpperCase().split(","); + parts.forEach((mode: string) => { + if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || + mode === "POSTGRESQL") { + this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); + } else if (mode === "ANSI_QUOTES") { + this.sqlModes.add(SqlMode.AnsiQuotes); + } else if (mode === "PIPES_AS_CONCAT") { + this.sqlModes.add(SqlMode.PipesAsConcat); + } else if (mode === "NO_BACKSLASH_ESCAPES") { + this.sqlModes.add(SqlMode.NoBackslashEscapes); + } else if (mode === "IGNORE_SPACE") { + this.sqlModes.add(SqlMode.IgnoreSpace); + } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { + this.sqlModes.add(SqlMode.HighNotPrecedence); + } + }); } } From 73924b64b8896cbe9961132bebaf30de4b3db8e9 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Fri, 29 Nov 2024 09:18:25 -0500 Subject: [PATCH 04/29] Clean up and make ts ports work. Apparently you cannot nest an enum in a class, so enums have to be global! What a screwed up OO language. --- sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts | 1 + sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts | 30 +-- sql/mysql/Oracle/Antlr4ng/SqlMode.ts | 12 +- sql/mysql/Oracle/Antlr4ng/SqlModes.ts | 36 +++ sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts | 3 +- .../Oracle/TypeScript/MySQLParserBase.ts | 3 +- sql/mysql/Oracle/TypeScript/SqlMode.ts | 12 +- sql/mysql/Oracle/TypeScript/SqlModes.ts | 36 +++ sql/mysql/Oracle/TypeScript/Test.ts | 229 ------------------ 9 files changed, 91 insertions(+), 271 deletions(-) create mode 100644 sql/mysql/Oracle/Antlr4ng/SqlModes.ts create mode 100644 sql/mysql/Oracle/TypeScript/SqlModes.ts delete mode 100644 sql/mysql/Oracle/TypeScript/Test.ts diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts index 307c113912..7de332f036 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts @@ -8,6 +8,7 @@ import { CharStream, Lexer, Token } from "antlr4ng"; import { MySQLLexer } from "./MySQLLexer.js"; import SqlMode from "./SqlMode.js"; +import SqlModes from "./SqlModes.js"; /** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ export abstract class MySQLLexerBase extends Lexer { diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts index 74ad3d8be7..89e637f844 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts @@ -4,6 +4,7 @@ import { Parser, TokenStream } from "antlr4ng"; import SqlMode from "./SqlMode.js"; +import SqlModes from "./SqlModes.js"; export abstract class MySQLParserBase extends Parser { @@ -17,7 +18,7 @@ export abstract class MySQLParserBase extends Parser { constructor(input: TokenStream) { super(input); this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** @@ -50,31 +51,4 @@ export abstract class MySQLParserBase extends Parser { { return this.serverVersion >= 80024 && this.serverVersion < 80031; } - - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public sqlModeFromString(modes: string): void { - this.sqlModes = new Set(); - - const parts = modes.toUpperCase().split(","); - parts.forEach((mode: string) => { - if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || - mode === "POSTGRESQL") { - this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); - } else if (mode === "ANSI_QUOTES") { - this.sqlModes.add(SqlMode.AnsiQuotes); - } else if (mode === "PIPES_AS_CONCAT") { - this.sqlModes.add(SqlMode.PipesAsConcat); - } else if (mode === "NO_BACKSLASH_ESCAPES") { - this.sqlModes.add(SqlMode.NoBackslashEscapes); - } else if (mode === "IGNORE_SPACE") { - this.sqlModes.add(SqlMode.IgnoreSpace); - } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { - this.sqlModes.add(SqlMode.HighNotPrecedence); - } - }); - } } diff --git a/sql/mysql/Oracle/Antlr4ng/SqlMode.ts b/sql/mysql/Oracle/Antlr4ng/SqlMode.ts index acebb55637..c1891b7b94 100644 --- a/sql/mysql/Oracle/Antlr4ng/SqlMode.ts +++ b/sql/mysql/Oracle/Antlr4ng/SqlMode.ts @@ -1,11 +1,11 @@ /** SQL modes that control parsing behavior. */ enum SqlMode { - NoMode, - AnsiQuotes, - HighNotPrecedence, - PipesAsConcat, - IgnoreSpace, - NoBackslashEscapes, + NoMode, + AnsiQuotes, + HighNotPrecedence, + PipesAsConcat, + IgnoreSpace, + NoBackslashEscapes } export default SqlMode; diff --git a/sql/mysql/Oracle/Antlr4ng/SqlModes.ts b/sql/mysql/Oracle/Antlr4ng/SqlModes.ts new file mode 100644 index 0000000000..178af2547c --- /dev/null +++ b/sql/mysql/Oracle/Antlr4ng/SqlModes.ts @@ -0,0 +1,36 @@ +/** SQL modes that control parsing behavior. */ + +import SqlMode from "./SqlMode.js"; + +export class SqlModes { + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public static sqlModeFromString(modes: string): Set { + var result = new Set(); + + const parts = modes.toUpperCase().split(","); + parts.forEach((mode: string) => { + if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || + mode === "POSTGRESQL") { + result.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); + } else if (mode === "ANSI_QUOTES") { + result.add(SqlMode.AnsiQuotes); + } else if (mode === "PIPES_AS_CONCAT") { + result.add(SqlMode.PipesAsConcat); + } else if (mode === "NO_BACKSLASH_ESCAPES") { + result.add(SqlMode.NoBackslashEscapes); + } else if (mode === "IGNORE_SPACE") { + result.add(SqlMode.IgnoreSpace); + } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { + result.add(SqlMode.HighNotPrecedence); + } + }); + return result; + } +} + +export default SqlModes; diff --git a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts index f9b3c80949..7482a4da26 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts @@ -9,6 +9,7 @@ import { CharStream, Lexer, Token } from "antlr4"; import { CommonToken } from "antlr4"; import MySQLLexer from "./MySQLLexer"; import SqlMode from "./SqlMode"; +import SqlModes from "./SqlModes"; /** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ export default abstract class MySQLLexerBase extends Lexer { @@ -36,7 +37,7 @@ export default abstract class MySQLLexerBase extends Lexer { constructor(input: CharStream) { super(input); this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** diff --git a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts index 0f6e1f0480..e802a7e3b5 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts @@ -5,6 +5,7 @@ import { Parser, TokenStream } from "antlr4"; import MySQLParser from "./MySQLParser"; import SqlMode from "./SqlMode"; +import SqlModes from "./SqlModes"; export default abstract class MySQLParserBase extends Parser { @@ -18,7 +19,7 @@ export default abstract class MySQLParserBase extends Parser { constructor(input: TokenStream) { super(input); this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** diff --git a/sql/mysql/Oracle/TypeScript/SqlMode.ts b/sql/mysql/Oracle/TypeScript/SqlMode.ts index acebb55637..c1891b7b94 100644 --- a/sql/mysql/Oracle/TypeScript/SqlMode.ts +++ b/sql/mysql/Oracle/TypeScript/SqlMode.ts @@ -1,11 +1,11 @@ /** SQL modes that control parsing behavior. */ enum SqlMode { - NoMode, - AnsiQuotes, - HighNotPrecedence, - PipesAsConcat, - IgnoreSpace, - NoBackslashEscapes, + NoMode, + AnsiQuotes, + HighNotPrecedence, + PipesAsConcat, + IgnoreSpace, + NoBackslashEscapes } export default SqlMode; diff --git a/sql/mysql/Oracle/TypeScript/SqlModes.ts b/sql/mysql/Oracle/TypeScript/SqlModes.ts new file mode 100644 index 0000000000..178af2547c --- /dev/null +++ b/sql/mysql/Oracle/TypeScript/SqlModes.ts @@ -0,0 +1,36 @@ +/** SQL modes that control parsing behavior. */ + +import SqlMode from "./SqlMode.js"; + +export class SqlModes { + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public static sqlModeFromString(modes: string): Set { + var result = new Set(); + + const parts = modes.toUpperCase().split(","); + parts.forEach((mode: string) => { + if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || + mode === "POSTGRESQL") { + result.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); + } else if (mode === "ANSI_QUOTES") { + result.add(SqlMode.AnsiQuotes); + } else if (mode === "PIPES_AS_CONCAT") { + result.add(SqlMode.PipesAsConcat); + } else if (mode === "NO_BACKSLASH_ESCAPES") { + result.add(SqlMode.NoBackslashEscapes); + } else if (mode === "IGNORE_SPACE") { + result.add(SqlMode.IgnoreSpace); + } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { + result.add(SqlMode.HighNotPrecedence); + } + }); + return result; + } +} + +export default SqlModes; diff --git a/sql/mysql/Oracle/TypeScript/Test.ts b/sql/mysql/Oracle/TypeScript/Test.ts deleted file mode 100644 index 584c83079e..0000000000 --- a/sql/mysql/Oracle/TypeScript/Test.ts +++ /dev/null @@ -1,229 +0,0 @@ -// Generated from trgen 0.23.7 - -import { CharStream } from 'antlr4'; -import { CharStreams } from 'antlr4'; -import { CommonTokenStream } from 'antlr4'; -import { ErrorListener } from 'antlr4'; -import { InputStream } from 'antlr4'; -import { Recognizer } from 'antlr4'; -import { RecognitionException } from 'antlr4'; -import { Token } from 'antlr4'; -import { readFileSync } from 'fs'; -import { writeFileSync } from 'fs'; -import { openSync } from 'fs'; -import { readSync } from 'fs'; -import { writeSync } from 'fs'; -import { closeSync } from 'fs'; -import { readFile } from 'fs/promises' - -import MySQLLexer from './MySQLLexer.js'; -import MySQLParser from './MySQLParser.js'; - -import { StringBuilder, emptyString, joinString, formatString, isNullOrWhiteSpace } from 'typescript-string-operations'; -import { Timer, Time, TimerOptions } from 'timer-node'; - - -function getChar() { - let buffer = Buffer.alloc(1); - var xx = 0; - try { - xx = readSync(0, buffer, 0, 1, null); - } catch (err) { - } - if (xx === 0) { - return ''; - } - return buffer.toString('utf8'); -} - - -class MyErrorListener extends ErrorListener { - _quiet: boolean; - _tee: boolean; - _output: any; - had_error: boolean; - - constructor(quiet: boolean, tee: boolean, output: any) { - super(); - this._quiet = quiet; - this._tee = tee; - this._output = output; - this.had_error = false; - } - - syntaxError(recognizer: Recognizer, offendingSymbol: T, line: number, column: number, msg: string, e: RecognitionException | undefined): void { - this.had_error = true; - if (this._tee) { - writeSync(this._output, `line ${line}:${column} ${msg}\n`); - } - if (!this._quiet) { - console.error(`line ${line}:${column} ${msg}`); - } - } -} - -var tee = false; -var show_profile = false; -var show_tree = false; -var show_tokens = false; -var show_trace = false; -var error_code = 0; -var quiet = false; -var encoding = 'utf8'; -var string_instance = 0; -var prefix = ''; -var inputs: string[] = []; -var is_fns: boolean[] = []; - -function splitLines(t: string) { return t.split(/\r\n|\r|\n/); } - -function main() { - for (let i = 2; i Date: Fri, 29 Nov 2024 11:56:08 -0500 Subject: [PATCH 05/29] Clean up the Java port. --- sql/mysql/Oracle/Cpp/SqlMode.java | 9 + sql/mysql/Oracle/Java/MySQLLexerBase.java | 2 + sql/mysql/Oracle/Java/MySQLParserBase.java | 2 + sql/mysql/Oracle/Java/SqlModes.java | 55 ++++ sql/mysql/Oracle/Java/Test.java | 291 --------------------- 5 files changed, 68 insertions(+), 291 deletions(-) create mode 100644 sql/mysql/Oracle/Cpp/SqlMode.java create mode 100644 sql/mysql/Oracle/Java/SqlModes.java delete mode 100644 sql/mysql/Oracle/Java/Test.java diff --git a/sql/mysql/Oracle/Cpp/SqlMode.java b/sql/mysql/Oracle/Cpp/SqlMode.java new file mode 100644 index 0000000000..80d9a50f70 --- /dev/null +++ b/sql/mysql/Oracle/Cpp/SqlMode.java @@ -0,0 +1,9 @@ +/** SQL modes that control parsing behavior. */ +public enum SqlMode { + NoMode, + AnsiQuotes, + HighNotPrecedence, + PipesAsConcat, + IgnoreSpace, + NoBackslashEscapes +} diff --git a/sql/mysql/Oracle/Java/MySQLLexerBase.java b/sql/mysql/Oracle/Java/MySQLLexerBase.java index fc6f7ae4a3..23d285a8b3 100644 --- a/sql/mysql/Oracle/Java/MySQLLexerBase.java +++ b/sql/mysql/Oracle/Java/MySQLLexerBase.java @@ -14,6 +14,8 @@ public abstract class MySQLLexerBase extends Lexer { public MySQLLexerBase(CharStream input) { super(input); + this.serverVersion = 80200; + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } public int serverVersion = 0; diff --git a/sql/mysql/Oracle/Java/MySQLParserBase.java b/sql/mysql/Oracle/Java/MySQLParserBase.java index e10dc207a0..352bbe233b 100644 --- a/sql/mysql/Oracle/Java/MySQLParserBase.java +++ b/sql/mysql/Oracle/Java/MySQLParserBase.java @@ -17,6 +17,8 @@ public abstract class MySQLParserBase extends Parser { protected MySQLParserBase(TokenStream input) { super(input); + this.serverVersion = 80200; + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** diff --git a/sql/mysql/Oracle/Java/SqlModes.java b/sql/mysql/Oracle/Java/SqlModes.java new file mode 100644 index 0000000000..179faa9b45 --- /dev/null +++ b/sql/mysql/Oracle/Java/SqlModes.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024, Oracle and/or its affiliates + */ + +/* eslint-disable no-underscore-dangle */ +/* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ + +import java.util.*; + +/** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ +public class SqlModes { + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public static Set sqlModeFromString(String modes) { + Set result = new HashSet(); + + String[] parts = modes.toUpperCase().split(","); + for (String mode : parts) { + switch (mode) { + case "ANSI": + case "DB2": + case "MAXDB": + case "MSSQL": + case "ORACLE": + case "POSTGRESQL": + result.add(SqlMode.AnsiQuotes); + result.add(SqlMode.PipesAsConcat); + result.add(SqlMode.IgnoreSpace); + break; + case "ANSI_QUOTES": + result.add(SqlMode.AnsiQuotes); + break; + case "PIPES_AS_CONCAT": + result.add(SqlMode.PipesAsConcat); + break; + case "NO_BACKSLASH_ESCAPES": + result.add(SqlMode.NoBackslashEscapes); + break; + case "IGNORE_SPACE": + result.add(SqlMode.IgnoreSpace); + break; + case "HIGH_NOT_PRECEDENCE": + case "MYSQL323": + case "MYSQL40": + result.add(SqlMode.HighNotPrecedence); + break; + } + } + return result; + } +} diff --git a/sql/mysql/Oracle/Java/Test.java b/sql/mysql/Oracle/Java/Test.java deleted file mode 100644 index 038b7a5f52..0000000000 --- a/sql/mysql/Oracle/Java/Test.java +++ /dev/null @@ -1,291 +0,0 @@ -// Generated from trgen 0.23.7 - -import java.io.FileNotFoundException; -import java.io.IOException; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.atn.*; -import org.antlr.v4.runtime.tree.ParseTree; -import java.time.Instant; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Scanner; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.io.OutputStreamWriter; -import java.io.FileOutputStream; -import java.nio.charset.StandardCharsets; -import java.io.File; -import org.antlr.v4.runtime.misc.*; -import org.antlr.v4.runtime.tree.*; - -public class Test { - - static boolean tee = false; - static boolean show_profile = false; - static boolean show_tree = false; - static boolean show_tokens = false; - static boolean show_trace = false; - static boolean show_diagnostic = false; - static int error_code = 0; - static java.nio.charset.Charset charset = null; - static int string_instance = 0; - static String prefix = ""; - static boolean quiet = false; - - public static void main(String[] args) throws FileNotFoundException, IOException - { - List is_fns = new ArrayList(); - List inputs = new ArrayList(); - for (int i = 0; i < args.length; ++i) - { - if (args[i].equals("-d")) - { - show_diagnostic = true; - } - else if (args[i].equals("-profile")) - { - show_profile = true; - } - else if (args[i].equals("-tokens")) - { - show_tokens = true; - } - else if (args[i].equals("-tree")) - { - show_tree = true; - } - else if (args[i].equals("-prefix")) - { - prefix = args[++i] + " "; - } - else if (args[i].equals("-input")) { - inputs.add(args[++i]); - is_fns.add(false); - } - else if (args[i].equals("-tee")) - { - tee = true; - } - else if (args[i].equals("-encoding")) - { - charset = java.nio.charset.Charset.forName(args[++i]); - } - else if (args[i].equals("-x")) - { - Scanner scanner = new Scanner(System.in); - for (; ; ) - { - if (!scanner.hasNext()) break; - String line = scanner.nextLine(); - if (line == null) break; - line = line.trim(); - if (line == "") break; - inputs.add(line); - is_fns.add(true); - } - } - else if (args[i].equals("-q")) - { - quiet = true; - } - else if (args[i].equals("-trace")) - { - show_trace = true; - continue; - } - else { - inputs.add(args[i]); - is_fns.add(true); - } - } - CharStream str = null; - if (inputs.size() == 0) - { - ParseStdin(); - } - else - { - Instant start = Instant.now(); - for (int f = 0; f < inputs.size(); ++f) - { - if (is_fns.get(f)) - ParseFilename(inputs.get(f), f); - else - ParseString(inputs.get(f), f); - } - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - if (!quiet) System.err.println("Total Time: " + (timeElapsed * 1.0) / 1000.0); - } - java.lang.System.exit(error_code); - } - - static void ParseStdin()throws IOException { - CharStream str = CharStreams.fromStream(System.in); - DoParse(str, "stdin", 0); - } - - static void ParseString(String input, int row_number) throws IOException { - var str = CharStreams.fromString(input); - DoParse(str, "string" + string_instance++, row_number); - } - - static void ParseFilename(String input, int row_number) throws IOException - { - CharStream str = null; - if (charset == null) - str = CharStreams.fromFileName(input); - else - str = CharStreams.fromFileName(input, charset); - DoParse(str, input, row_number); - } - - static void DoParse(CharStream str, String input_name, int row_number) { - MySQLLexer lexer = new MySQLLexer(str); - lexer.serverVersion = 80200; - lexer.sqlModeFromString("ANSI_QUOTES"); - if (show_tokens) - { - StringBuilder new_s = new StringBuilder(); - for (int i = 0; ; ++i) - { - var ro_token = lexer.nextToken(); - var token = (CommonToken)ro_token; - token.setTokenIndex(i); - new_s.append(token.toString()); - new_s.append(System.getProperty("line.separator")); - if (token.getType() == IntStream.EOF) - break; - } - System.err.println(new_s.toString()); - lexer.reset(); - } - var tokens = new CommonTokenStream(lexer); - MySQLParser parser = new MySQLParser(tokens); -//lexer.charSets = charSets; - parser.serverVersion = lexer.serverVersion; - parser.sqlModes = lexer.sqlModes; - PrintStream output = null; - try { - output = tee ? new PrintStream(new File(input_name + ".errors")) : System.out; - } catch (NullPointerException e) { - output = System.err; - } catch (FileNotFoundException e2) { - output = System.err; - } - ErrorListener listener_lexer = new ErrorListener(quiet, tee, output); - ErrorListener listener_parser = new ErrorListener(quiet, tee, output); - parser.removeErrorListeners(); - lexer.removeErrorListeners(); - parser.addErrorListener(listener_parser); - lexer.addErrorListener(listener_lexer); - if (show_diagnostic) - { - parser.addErrorListener(new MyDiagnosticErrorListener()); - } - if (show_trace) - { - parser.setTrace(true); -// ParserATNSimulator.trace_atn_sim = true; - } - Instant start = Instant.now(); - ParseTree tree = parser.queries(); - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - String result = ""; - if (listener_parser.had_error || listener_lexer.had_error) - { - result = "fail"; - error_code = 1; - } - else - result = "success"; - if (show_tree) - { - if (tee) - { - PrintWriter treef = null; - try { - treef = new PrintWriter(new OutputStreamWriter(new FileOutputStream(new File(input_name + ".tree")), StandardCharsets.UTF_8), true); - //treef = new PrintStream(new File(input_name + ".tree")); - } catch (NullPointerException e) { - treef = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true);; - } catch (FileNotFoundException e2) { - treef = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true); - } - treef.print(tree.toStringTree(parser)); - treef.close(); - } else - { - System.err.println(tree.toStringTree(parser)); - } - } - if (!quiet) - { - System.err.println(prefix + "Java " + row_number + " " + input_name + " " + result + " " + (timeElapsed * 1.0) / 1000.0); - } - if (tee) output.close(); - } - - public static String toStringTree(Tree tree, Parser recog) { - StringBuilder sb = new StringBuilder(); - String[] ruleNames = recog != null ? recog.getRuleNames() : null; - List ruleNamesList = ruleNames != null ? List.of(ruleNames) : null; - toStringTree(sb, tree, 0, ruleNamesList); - return sb.toString(); - } - - public static void toStringTree(StringBuilder sb, Tree t, int indent, List ruleNames) { - String s = org.antlr.v4.runtime.misc.Utils.escapeWhitespace(getNodeText(t, ruleNames), false); - if (t.getChildCount() == 0) { - for (int i = 0; i < indent; ++i) sb.append(" "); - sb.append(s).append(System.lineSeparator()); - return; - } - s = org.antlr.v4.runtime.misc.Utils.escapeWhitespace(getNodeText(t, ruleNames), false); - for (int i = 0; i < indent; ++i) sb.append(' '); - sb.append(s).append(System.lineSeparator()); - for (int i = 0; i < t.getChildCount(); i++) { - toStringTree(sb, t.getChild(i), indent + 1, ruleNames); - } - } - - public static String getNodeText(Tree t, Parser recog) { - String[] ruleNames = recog != null ? recog.getRuleNames() : null; - List ruleNamesList = ruleNames != null ? java.util.Arrays.asList(ruleNames) : null; - return getNodeText(t, ruleNamesList); - } - - public static String getNodeText(Tree t, List ruleNames) { - if ( ruleNames!=null ) { - if ( t instanceof RuleContext ) { - int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); - String ruleName = ruleNames.get(ruleIndex); - int altNumber = ((RuleContext) t).getAltNumber(); - if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { - return ruleName+":"+altNumber; - } - return ruleName; - } - else if ( t instanceof ErrorNode) { - return t.toString(); - } - else if ( t instanceof TerminalNode) { - Token symbol = ((TerminalNode)t).getSymbol(); - if (symbol != null) { - String s = symbol.getText(); - return s; - } - } - } - // no recog for rule names - Object payload = t.getPayload(); - if ( payload instanceof Token ) { - return ((Token)payload).getText(); - } - return t.getPayload().toString(); - } - -} From 8bf6c26c8f30fee09b64730dccf1f02d106561a3 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Fri, 29 Nov 2024 13:51:05 -0500 Subject: [PATCH 06/29] Add in Cpp port (incomplete). --- sql/mysql/Oracle/Cpp/MySQLLexerBase.h | 121 ++++++++++++++++++ sql/mysql/Oracle/Cpp/MySQLParserBase.h | 41 ++++++ .../Oracle/Cpp/{SqlMode.java => SqlMode.h} | 6 +- sql/mysql/Oracle/Cpp/SqlModes.h | 14 ++ sql/mysql/Oracle/Cpp/transformGrammar.py | 34 +++++ 5 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 sql/mysql/Oracle/Cpp/MySQLLexerBase.h create mode 100644 sql/mysql/Oracle/Cpp/MySQLParserBase.h rename sql/mysql/Oracle/Cpp/{SqlMode.java => SqlMode.h} (83%) create mode 100644 sql/mysql/Oracle/Cpp/SqlModes.h create mode 100644 sql/mysql/Oracle/Cpp/transformGrammar.py diff --git a/sql/mysql/Oracle/Cpp/MySQLLexerBase.h b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h new file mode 100644 index 0000000000..eea7adedfe --- /dev/null +++ b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h @@ -0,0 +1,121 @@ +/* + * Copyright 2024, Oracle and/or its affiliates + */ + +/* eslint-disable no-underscore-dangle */ +/* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ + +#pragma once +#include "antlr4-runtime.h" +#include +#include "SqlMode.h" +#include + +/** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ +class MySQLLexerBase : public antlr4::Lexer +{ + + public: + MySQLLexerBase(antlr4::CharStream * input); + int serverVersion; + std::set sqlModes; + + /** Enable Multi Language Extension support. */ + bool supportMle; + + std::set charSets; + + protected: + bool inVersionComment; + + private: + std::queue pendingTokens; + static std::string longString; + static int longLength; + static std::string signedLongString; + static std::string longLongString; + static int longLongLength; + static std::string signedLongLongString; + static int signedLongLongLength; + static std::string unsignedLongLongString; + static int unsignedLongLongLength; + + bool justEmitedDot; + + /** + * Determines if the given SQL mode is currently active in the lexer. + * + * @param mode The mode to check. + * + * @returns True if the mode is one of the currently active modes. + */ + public: + bool isSqlModeActive(SqlMode mode); + void reset(); + std::unique_ptr nextToken() override; + + protected: + bool checkMySQLVersion(std::string text); + int determineFunction(int proposed); + int determineNumericType(std::string text); + int checkCharset(std::string text); + void emitDot(); + + public: + bool isServerVersionLt80024(); + bool isServerVersionGe80024(); + bool isServerVersionGe80011(); + bool isServerVersionGe80013(); + bool isServerVersionLt80014(); + bool isServerVersionGe80014(); + bool isServerVersionGe80017(); + bool isServerVersionGe80018(); + bool isMasterCompressionAlgorithm(); + bool isServerVersionLt80031(); + void doLogicalOr(); + void doIntNumber(); + void doAdddate(); + void doBitAnd(); + void doBitOr(); + void doBitXor(); + void doCast(); + void doCount(); + void doCurdate(); + void doCurrentDate(); + void doCurrentTime(); + void doCurtime(); + void doDateAdd(); + void doDateSub(); + void doExtract(); + void doGroupConcat(); + void doMax(); + void doMid(); + void doMin(); + void doNot(); + void doNow(); + void doPosition(); + void doSessionUser(); + void doStddevSamp(); + void doStddev(); + void doStddevPop(); + void doStd(); + void doSubdate(); + void doSubstr(); + void doSubstring(); + void doSum(); + void doSysdate(); + void doSystemUser(); + void doTrim(); + void doVariance(); + void doVarPop(); + void doVarSamp(); + void doUnderscoreCharset(); + bool isVersionComment(); + bool isBackTickQuotedId(); + bool isDoubleQuotedText(); + bool isSingleQuotedText(); + antlr4::Token* emit() override; + void startInVersionComment(); + void endInVersionComment(); + bool isInVersionComment(); +}; diff --git a/sql/mysql/Oracle/Cpp/MySQLParserBase.h b/sql/mysql/Oracle/Cpp/MySQLParserBase.h new file mode 100644 index 0000000000..a31e551d8e --- /dev/null +++ b/sql/mysql/Oracle/Cpp/MySQLParserBase.h @@ -0,0 +1,41 @@ +/* + * Copyright 2024, Oracle and/or its affiliates + */ + +/* eslint-disable no-underscore-dangle */ +/* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ + +#pragma once +#include "antlr4-runtime.h" +#include +#include "SqlMode.h" +#include + +class MySQLParserBase : public antlr4::Parser { + + // To parameterize the parsing process. + public: + int serverVersion; + std::set sqlModes; + + public: + /** Enable Multi Language Extension support. */ + bool supportMle; + + protected: + MySQLParserBase(antlr4::TokenStream* input); + + public: + /** + * Determines if the given SQL mode is currently active in the lexer. + * + * @param mode The mode to check. + * + * @returns True if the mode is one of the currently active modes. + */ + bool isSqlModeActive(SqlMode mode); + bool isPureIdentifier(); + bool isTextStringLiteral(); + bool isStoredRoutineBody(); + bool isSelectStatementWithInto(); +}; diff --git a/sql/mysql/Oracle/Cpp/SqlMode.java b/sql/mysql/Oracle/Cpp/SqlMode.h similarity index 83% rename from sql/mysql/Oracle/Cpp/SqlMode.java rename to sql/mysql/Oracle/Cpp/SqlMode.h index 80d9a50f70..260a91cb95 100644 --- a/sql/mysql/Oracle/Cpp/SqlMode.java +++ b/sql/mysql/Oracle/Cpp/SqlMode.h @@ -1,9 +1,11 @@ +#pragma once + /** SQL modes that control parsing behavior. */ -public enum SqlMode { +enum SqlMode { NoMode, AnsiQuotes, HighNotPrecedence, PipesAsConcat, IgnoreSpace, NoBackslashEscapes -} +}; diff --git a/sql/mysql/Oracle/Cpp/SqlModes.h b/sql/mysql/Oracle/Cpp/SqlModes.h new file mode 100644 index 0000000000..d1b24a3717 --- /dev/null +++ b/sql/mysql/Oracle/Cpp/SqlModes.h @@ -0,0 +1,14 @@ +/** SQL modes that control parsing behavior. */ +#include once +#include "SqlMode.h" + +class SqlModes { + + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public: + static std::set sqlModeFromString(std::string modes); +} diff --git a/sql/mysql/Oracle/Cpp/transformGrammar.py b/sql/mysql/Oracle/Cpp/transformGrammar.py new file mode 100644 index 0000000000..ab3635dcda --- /dev/null +++ b/sql/mysql/Oracle/Cpp/transformGrammar.py @@ -0,0 +1,34 @@ +import sys, os, re, shutil +from glob import glob +from pathlib import Path + +def main(argv): + for file in glob("./*.g4"): + fix(file) + +def fix(file_path): + print("Altering " + file_path) + if not os.path.exists(file_path): + print(f"Could not find file: {file_path}") + sys.exit(1) + parts = os.path.split(file_path) + file_name = parts[-1] + shutil.move(file_path, file_path + ".bak") + input_file = open(file_path + ".bak",'r') + output_file = open(file_path, 'w') + for x in input_file: + if '// Insert here @header for lexer.' in x: + x = x.replace('// Insert here @header for lexer.', '@header {#include "MySQLLexerBase.h"}') + if '// Insert here @header for parser.' in x: + x = x.replace('// Insert here @header for parser.', '@header {#include "MySQLParserBase.h"}') + if 'this.' in x: + x = x.replace('this.', 'this->') + output_file.write(x) + output_file.flush() + + print("Writing ...") + input_file.close() + output_file.close() + +if __name__ == '__main__': + main(sys.argv) From 334e994b9005680a8fcb7aebf01c5d3615db4261 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sat, 30 Nov 2024 14:25:21 -0500 Subject: [PATCH 07/29] Clean up, add Cpp target. --- sql/mysql/Oracle/CSharp/MySQLParserBase.cs | 2 +- sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp | 426 ++++++++++++++++++ sql/mysql/Oracle/Cpp/MySQLLexerBase.h | 4 +- sql/mysql/Oracle/Cpp/MySQLParserBase.h | 3 +- sql/mysql/Oracle/Cpp/SqlModes.cpp | 34 ++ sql/mysql/Oracle/Cpp/SqlModes.h | 10 +- sql/mysql/Oracle/Dart/MySQLLexerBase.dart | 4 +- sql/mysql/Oracle/Java/MySQLLexerBase.java | 42 -- sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts | 27 -- .../Oracle/TypeScript/MySQLParserBase.ts | 27 -- sql/mysql/Oracle/desc.xml | 2 +- 11 files changed, 475 insertions(+), 106 deletions(-) create mode 100644 sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp create mode 100644 sql/mysql/Oracle/Cpp/SqlModes.cpp diff --git a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs index 7c7274eb11..692d701c88 100644 --- a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs @@ -18,7 +18,7 @@ public abstract class MySQLParserBase : Parser { protected MySQLParserBase(ITokenStream input, TextWriter output, TextWriter errorOutput) : base(input, output, errorOutput) { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModeFromString("ANSI_QUOTES"); } /** diff --git a/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp b/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp new file mode 100644 index 0000000000..d8656b56c3 --- /dev/null +++ b/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp @@ -0,0 +1,426 @@ +#include "antlr4-runtime.h" +#include "MySQLLexerBase.h" +#include "MySQLLexer.h" +#include "SqlMode.h" +#include "SqlModes.h" + +#include +#include +#include +#include +#include + +MySQLLexerBase::MySQLLexerBase(antlr4::CharStream * input) : antlr4::Lexer(input) +{ + this->serverVersion = 80200; + this->sqlModes = SqlModes::sqlModeFromString("ANSI_QUOTES"); + this->supportMle = true; + this->inVersionComment = false; +} + +bool MySQLLexerBase::isSqlModeActive(SqlMode mode) { + return sqlModes.count(mode) > 0; +} + +void MySQLLexerBase::reset() { + inVersionComment = false; + Lexer::reset(); +} + +std::unique_ptr MySQLLexerBase::nextToken() +{ + if (!this->pendingTokens.empty()) { + std::unique_ptr pending = std::move(this->pendingTokens.front()); + this->pendingTokens.pop(); + return std::move(pending); + } + + std::unique_ptr next = std::move(Lexer::nextToken()); + if (!pendingTokens.empty()) { + std::unique_ptr pending = std::move(this->pendingTokens.front()); + this->pendingTokens.pop(); + this->pendingTokens.push(std::move(next)); + return std::move(pending); + } + + return std::move(next); +} + +bool MySQLLexerBase::checkMySQLVersion(std::string text) { + if (text.length() < 8) return false; + + int version = std::stoi(text.substr(3)); + if (version <= serverVersion) { + inVersionComment = true; + return true; + } + + return false; +} + +int MySQLLexerBase::determineFunction(int proposed) { + char input = static_cast(_input->LA(1)); + if (isSqlModeActive(SqlMode::IgnoreSpace)) { + while (isspace(input)) { + this->getInterpreter()->consume(_input); + input = static_cast(_input->LA(1)); + } + } + + return (input == '(') ? proposed : MySQLLexer::IDENTIFIER; +} + +int MySQLLexerBase::determineNumericType(std::string text) { + int length = text.length() - 1; + if (length < longLength) return MySQLLexer::INT_NUMBER; + + bool negative = false; + int index = 0; + + if (text[index] == '+') { + ++index; --length; + } else if (text[index] == '-') { + ++index; --length; + negative = true; + } + + while (text[index] == '0' && length > 0) { + ++index; --length; + } + + if (length < longLength) return MySQLLexer::INT_NUMBER; + + std::string cmp; + int smaller, bigger; + + if (negative) { + if (length == longLength) { + cmp = signedLongString.substr(1); + smaller = MySQLLexer::INT_NUMBER; + bigger = MySQLLexer::LONG_NUMBER; + } else if (length < signedLongLongLength) { + return MySQLLexer::LONG_NUMBER; + } else if (length > signedLongLongLength) { + return MySQLLexer::DECIMAL_NUMBER; + } else { + cmp = signedLongLongString.substr(1); + smaller = MySQLLexer::LONG_NUMBER; + bigger = MySQLLexer::DECIMAL_NUMBER; + } + } else { + if (length == longLength) { + cmp = longString; + smaller = MySQLLexer::INT_NUMBER; + bigger = MySQLLexer::LONG_NUMBER; + } else if (length < longLongLength) { + return MySQLLexer::LONG_NUMBER; + } else if (length > longLongLength) { + if (length > unsignedLongLongLength) return MySQLLexer::DECIMAL_NUMBER; + cmp = unsignedLongLongString; + smaller = MySQLLexer::ULONGLONG_NUMBER; + bigger = MySQLLexer::DECIMAL_NUMBER; + } else { + cmp = longLongString; + smaller = MySQLLexer::LONG_NUMBER; + bigger = MySQLLexer::ULONGLONG_NUMBER; + } + } + + int cmpIndex = 0; + while (index < text.length() && text[index] == cmp[cmpIndex]) { + ++index; ++cmpIndex; + } + + return (text[index - 1] <= cmp[cmpIndex - 1]) ? smaller : bigger; +} + +int MySQLLexerBase::checkCharset(std::string text) { + return (charSets.count(text) > 0) ? MySQLLexer::UNDERSCORE_CHARSET : MySQLLexer::IDENTIFIER; +} + +void MySQLLexerBase::emitDot() { + auto * t = new antlr4::CommonToken( + std::pair(this, this->_input), + MySQLLexer::DOT_SYMBOL, 0, this->tokenStartCharIndex, this->tokenStartCharIndex); + pendingTokens.push(std::make_unique(t)); + this->setCharPositionInLine(this->getCharPositionInLine() + 1); + + ++this->tokenStartCharIndex; +} + +bool MySQLLexerBase::isServerVersionLt80024() +{ + return this->serverVersion < 80024; + +} + +bool MySQLLexerBase::isServerVersionGe80024() +{ + return this->serverVersion >= 80024; +} + +bool MySQLLexerBase::isServerVersionGe80011() +{ + return this->serverVersion >= 80011; +} + +bool MySQLLexerBase::isServerVersionGe80013() +{ + return this->serverVersion >= 80013; +} + +bool MySQLLexerBase::isServerVersionLt80014() +{ + return this->serverVersion < 80014; +} + +bool MySQLLexerBase::isServerVersionGe80014() +{ + return this->serverVersion >= 80014; +} + +bool MySQLLexerBase::isServerVersionGe80017() +{ + return this->serverVersion >= 80017; +} + +bool MySQLLexerBase::isServerVersionGe80018() { return this->serverVersion >= 80018; } + +bool MySQLLexerBase::isMasterCompressionAlgorithm() { return this->serverVersion >= 80018 && this->isServerVersionLt80024(); } + +bool MySQLLexerBase::isServerVersionLt80031() { + return this->serverVersion < 80031; +} + +void MySQLLexerBase::doLogicalOr() { + this->type = this->isSqlModeActive(SqlMode::PipesAsConcat) ? MySQLLexer::CONCAT_PIPES_SYMBOL : MySQLLexer::LOGICAL_OR_OPERATOR; +} + +void MySQLLexerBase::doIntNumber() +{ + this->type = this->determineNumericType(this->getText()); +} + +void MySQLLexerBase::doAdddate() +{ + this->type = this->determineFunction(MySQLLexer::ADDDATE_SYMBOL); +} + +void MySQLLexerBase::doBitAnd() +{ + this->type = this->determineFunction(MySQLLexer::BIT_AND_SYMBOL); +} + +void MySQLLexerBase::doBitOr() +{ + this->type = this->determineFunction(MySQLLexer::BIT_OR_SYMBOL); +} + +void MySQLLexerBase::doBitXor() +{ + this->type = this->determineFunction(MySQLLexer::BIT_XOR_SYMBOL); +} + +void MySQLLexerBase::doCast() +{ + this->type = this->determineFunction(MySQLLexer::CAST_SYMBOL); +} + +void MySQLLexerBase::doCount() +{ + this->type = this->determineFunction(MySQLLexer::COUNT_SYMBOL); +} + +void MySQLLexerBase::doCurdate() +{ + this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); +} + +void MySQLLexerBase::doCurrentDate() +{ + this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); +} + +void MySQLLexerBase::doCurrentTime() +{ + this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); +} + +void MySQLLexerBase::doCurtime() +{ + this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); +} + +void MySQLLexerBase::doDateAdd() +{ + this->type = this->determineFunction(MySQLLexer::DATE_ADD_SYMBOL); +} + +void MySQLLexerBase::doDateSub() +{ + this->type = this->determineFunction(MySQLLexer::DATE_SUB_SYMBOL); +} + +void MySQLLexerBase::doExtract() +{ + this->type = this->determineFunction(MySQLLexer::EXTRACT_SYMBOL); +} + +void MySQLLexerBase::doGroupConcat() +{ + this->type = this->determineFunction(MySQLLexer::GROUP_CONCAT_SYMBOL); +} + +void MySQLLexerBase::doMax() +{ + this->type = this->determineFunction(MySQLLexer::MAX_SYMBOL); +} + +void MySQLLexerBase::doMid() +{ + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); +} + +void MySQLLexerBase::doMin() +{ + this->type = this->determineFunction(MySQLLexer::MIN_SYMBOL); +} + +void MySQLLexerBase::doNot() +{ + this->type = this->isSqlModeActive(SqlMode::HighNotPrecedence) ? MySQLLexer::NOT2_SYMBOL: MySQLLexer::NOT_SYMBOL; +} + +void MySQLLexerBase::doNow() +{ + this->type = this->determineFunction(MySQLLexer::NOW_SYMBOL); +} + +void MySQLLexerBase::doPosition() +{ + this->type = this->determineFunction(MySQLLexer::POSITION_SYMBOL); +} + +void MySQLLexerBase::doSessionUser() +{ + this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); +} + +void MySQLLexerBase::doStddevSamp() +{ + this->type = this->determineFunction(MySQLLexer::STDDEV_SAMP_SYMBOL); +} + +void MySQLLexerBase::doStddev() +{ + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); +} + +void MySQLLexerBase::doStddevPop() +{ + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); +} + +void MySQLLexerBase::doStd() +{ + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); +} + +void MySQLLexerBase::doSubdate() +{ + this->type = this->determineFunction(MySQLLexer::SUBDATE_SYMBOL); +} + +void MySQLLexerBase::doSubstr() +{ + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); +} + +void MySQLLexerBase::doSubstring() +{ + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); +} + +void MySQLLexerBase::doSum() +{ + this->type = this->determineFunction(MySQLLexer::SUM_SYMBOL); +} + +void MySQLLexerBase::doSysdate() +{ + this->type = this->determineFunction(MySQLLexer::SYSDATE_SYMBOL); +} + +void MySQLLexerBase::doSystemUser() +{ + this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); +} + +void MySQLLexerBase::doTrim() +{ + this->type = this->determineFunction(MySQLLexer::TRIM_SYMBOL); +} + +void MySQLLexerBase::doVariance() +{ + this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); +} + +void MySQLLexerBase::doVarPop() +{ + this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); +} + +void MySQLLexerBase::doVarSamp() +{ + this->type = this->determineFunction(MySQLLexer::VAR_SAMP_SYMBOL); +} + +void MySQLLexerBase::doUnderscoreCharset() +{ + this->type = this->checkCharset(this->getText()); +} + +bool MySQLLexerBase::isVersionComment() { + return inVersionComment; +} + +bool MySQLLexerBase::isBackTickQuotedId() +{ + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); +} + +bool MySQLLexerBase::isDoubleQuotedText() +{ + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); +} + +bool MySQLLexerBase::isSingleQuotedText() +{ + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); +} + +void MySQLLexerBase::startInVersionComment() +{ + inVersionComment = true; +} + +void MySQLLexerBase::endInVersionComment() +{ + inVersionComment = false; +} + +bool MySQLLexerBase::isInVersionComment() +{ + return inVersionComment; +} + +std::string MySQLLexerBase::longString = "2147483647"; +int MySQLLexerBase::longLength = 10; +std::string MySQLLexerBase::signedLongString = "-2147483648"; +std::string MySQLLexerBase::longLongString = "9223372036854775807"; +int MySQLLexerBase::longLongLength = 19; +std::string MySQLLexerBase::signedLongLongString = "-9223372036854775808"; +int MySQLLexerBase::signedLongLongLength = 19; +std::string MySQLLexerBase::unsignedLongLongString = "18446744073709551615"; +int MySQLLexerBase::unsignedLongLongLength = 20; diff --git a/sql/mysql/Oracle/Cpp/MySQLLexerBase.h b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h index eea7adedfe..3f075e6ff6 100644 --- a/sql/mysql/Oracle/Cpp/MySQLLexerBase.h +++ b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h @@ -6,6 +6,7 @@ /* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ #pragma once + #include "antlr4-runtime.h" #include #include "SqlMode.h" @@ -29,7 +30,7 @@ class MySQLLexerBase : public antlr4::Lexer bool inVersionComment; private: - std::queue pendingTokens; + std::queue> pendingTokens; static std::string longString; static int longLength; static std::string signedLongString; @@ -114,7 +115,6 @@ class MySQLLexerBase : public antlr4::Lexer bool isBackTickQuotedId(); bool isDoubleQuotedText(); bool isSingleQuotedText(); - antlr4::Token* emit() override; void startInVersionComment(); void endInVersionComment(); bool isInVersionComment(); diff --git a/sql/mysql/Oracle/Cpp/MySQLParserBase.h b/sql/mysql/Oracle/Cpp/MySQLParserBase.h index a31e551d8e..2fdc73e419 100644 --- a/sql/mysql/Oracle/Cpp/MySQLParserBase.h +++ b/sql/mysql/Oracle/Cpp/MySQLParserBase.h @@ -6,6 +6,7 @@ /* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ #pragma once + #include "antlr4-runtime.h" #include #include "SqlMode.h" @@ -22,7 +23,7 @@ class MySQLParserBase : public antlr4::Parser { /** Enable Multi Language Extension support. */ bool supportMle; - protected: + protected: MySQLParserBase(antlr4::TokenStream* input); public: diff --git a/sql/mysql/Oracle/Cpp/SqlModes.cpp b/sql/mysql/Oracle/Cpp/SqlModes.cpp new file mode 100644 index 0000000000..62714dbfd5 --- /dev/null +++ b/sql/mysql/Oracle/Cpp/SqlModes.cpp @@ -0,0 +1,34 @@ +#include "SqlMode.h" +#include "SqlModes.h" + +#include +#include +#include +#include +#include + +std::set SqlModes::sqlModeFromString(const std::string& modes) { + std::set result; + std::stringstream ss(modes); + std::string mode; + + while (std::getline(ss, mode, ',')) { + std::transform(mode.begin(), mode.end(), mode.begin(), ::toupper); + if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || + mode == "ORACLE" || mode == "POSTGRESQL") { + result.insert({ SqlMode::AnsiQuotes, SqlMode::PipesAsConcat, SqlMode::IgnoreSpace }); + } else if (mode == "ANSI_QUOTES") { + result.insert(SqlMode::AnsiQuotes); + } else if (mode == "PIPES_AS_CONCAT") { + result.insert(SqlMode::PipesAsConcat); + } else if (mode == "NO_BACKSLASH_ESCAPES") { + result.insert(SqlMode::NoBackslashEscapes); + } else if (mode == "IGNORE_SPACE") { + result.insert(SqlMode::IgnoreSpace); + } else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40") { + result.insert(SqlMode::HighNotPrecedence); + } + } + return result; +} + diff --git a/sql/mysql/Oracle/Cpp/SqlModes.h b/sql/mysql/Oracle/Cpp/SqlModes.h index d1b24a3717..834eaffc63 100644 --- a/sql/mysql/Oracle/Cpp/SqlModes.h +++ b/sql/mysql/Oracle/Cpp/SqlModes.h @@ -1,7 +1,11 @@ /** SQL modes that control parsing behavior. */ -#include once +#pragma once + #include "SqlMode.h" +#include +#include + class SqlModes { /** @@ -10,5 +14,5 @@ class SqlModes { * @param modes The input string to parse. */ public: - static std::set sqlModeFromString(std::string modes); -} + static std::set sqlModeFromString(const std::string& modes); +}; diff --git a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart index 7544f31ab0..f78a3e402c 100644 --- a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart +++ b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart @@ -92,7 +92,7 @@ abstract class MySQLLexerBase extends Lexer { // First respond with pending tokens to the next token request, if there are any. if (! this.pendingTokens.isEmpty) { - var pending = this.pendingTokens.removeFirst(); + var pending = this.pendingTokens.removeFirst(); return pending; } @@ -101,7 +101,7 @@ abstract class MySQLLexerBase extends Lexer var next = super.nextToken(); if (! this.pendingTokens.isEmpty) { - var pending = this.pendingTokens.removeFirst(); + var pending = this.pendingTokens.removeFirst(); this.pendingTokens.add(next); return pending; } diff --git a/sql/mysql/Oracle/Java/MySQLLexerBase.java b/sql/mysql/Oracle/Java/MySQLLexerBase.java index 23d285a8b3..44a3959ced 100644 --- a/sql/mysql/Oracle/Java/MySQLLexerBase.java +++ b/sql/mysql/Oracle/Java/MySQLLexerBase.java @@ -52,48 +52,6 @@ public boolean isSqlModeActive(SqlMode mode) { return this.sqlModes.contains(mode); } - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public void sqlModeFromString(String modes) { - this.sqlModes = new HashSet<>(); - - String[] parts = modes.toUpperCase().split(","); - for (String mode : parts) { - switch (mode) { - case "ANSI": - case "DB2": - case "MAXDB": - case "MSSQL": - case "ORACLE": - case "POSTGRESQL": - this.sqlModes.add(SqlMode.AnsiQuotes); - this.sqlModes.add(SqlMode.PipesAsConcat); - this.sqlModes.add(SqlMode.IgnoreSpace); - break; - case "ANSI_QUOTES": - this.sqlModes.add(SqlMode.AnsiQuotes); - break; - case "PIPES_AS_CONCAT": - this.sqlModes.add(SqlMode.PipesAsConcat); - break; - case "NO_BACKSLASH_ESCAPES": - this.sqlModes.add(SqlMode.NoBackslashEscapes); - break; - case "IGNORE_SPACE": - this.sqlModes.add(SqlMode.IgnoreSpace); - break; - case "HIGH_NOT_PRECEDENCE": - case "MYSQL323": - case "MYSQL40": - this.sqlModes.add(SqlMode.HighNotPrecedence); - break; - } - } - } - @Override public void reset() { this.inVersionComment = false; diff --git a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts index 7482a4da26..36e07da86a 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts @@ -51,33 +51,6 @@ export default abstract class MySQLLexerBase extends Lexer { return this.sqlModes.has(mode); } - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public sqlModeFromString(modes: string): void { - this.sqlModes = new Set(); - - const parts = modes.toUpperCase().split(","); - parts.forEach((mode: string) => { - if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || - mode === "POSTGRESQL") { - this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); - } else if (mode === "ANSI_QUOTES") { - this.sqlModes.add(SqlMode.AnsiQuotes); - } else if (mode === "PIPES_AS_CONCAT") { - this.sqlModes.add(SqlMode.PipesAsConcat); - } else if (mode === "NO_BACKSLASH_ESCAPES") { - this.sqlModes.add(SqlMode.NoBackslashEscapes); - } else if (mode === "IGNORE_SPACE") { - this.sqlModes.add(SqlMode.IgnoreSpace); - } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { - this.sqlModes.add(SqlMode.HighNotPrecedence); - } - }); - } - /** * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. */ diff --git a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts index e802a7e3b5..5744763f15 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts @@ -52,31 +52,4 @@ export default abstract class MySQLParserBase extends Parser { { return this.serverVersion >= 80024 && this.serverVersion < 80031; } - - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public sqlModeFromString(modes: string): void { - this.sqlModes = new Set(); - - const parts = modes.toUpperCase().split(","); - parts.forEach((mode: string) => { - if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || - mode === "POSTGRESQL") { - this.sqlModes.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); - } else if (mode === "ANSI_QUOTES") { - this.sqlModes.add(SqlMode.AnsiQuotes); - } else if (mode === "PIPES_AS_CONCAT") { - this.sqlModes.add(SqlMode.PipesAsConcat); - } else if (mode === "NO_BACKSLASH_ESCAPES") { - this.sqlModes.add(SqlMode.NoBackslashEscapes); - } else if (mode === "IGNORE_SPACE") { - this.sqlModes.add(SqlMode.IgnoreSpace); - } else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { - this.sqlModes.add(SqlMode.HighNotPrecedence); - } - }); - } } diff --git a/sql/mysql/Oracle/desc.xml b/sql/mysql/Oracle/desc.xml index 78656ebb33..7dbffd7764 100644 --- a/sql/mysql/Oracle/desc.xml +++ b/sql/mysql/Oracle/desc.xml @@ -1,5 +1,5 @@ - Antlr4ng;CSharp;Dart;Java;Python3;TypeScript + Antlr4ng;Cpp;CSharp;Dart;Java;Python3;TypeScript examples/**/*.sql From 10bc14de8ca3e38c29b309702d97842ab1205528 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sat, 30 Nov 2024 14:34:26 -0500 Subject: [PATCH 08/29] Forgot to add this new file. --- sql/mysql/Oracle/Cpp/MySQLParserBase.cpp | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 sql/mysql/Oracle/Cpp/MySQLParserBase.cpp diff --git a/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp b/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp new file mode 100644 index 0000000000..8306fae146 --- /dev/null +++ b/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp @@ -0,0 +1,48 @@ +/* + * Copyright 2024, Oracle and/or its affiliates + */ + +/* eslint-disable no-underscore-dangle */ +/* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ + +#pragma once + +#include +#include + +#include "antlr4-runtime.h" +#include "SqlMode.h" +#include "SqlModes.h" +#include "MySQLParserBase.h" + + +MySQLParserBase::MySQLParserBase(antlr4::TokenStream* input) : Parser(input) +{ + this->serverVersion = 80200; + this->sqlModes = SqlModes::sqlModeFromString("ANSI_QUOTES"); +} + +bool MySQLParserBase::isSqlModeActive(SqlMode mode) +{ + return sqlModes.count(mode) > 0; +} + +bool MySQLParserBase::isPureIdentifier() +{ + return this->isSqlModeActive(SqlMode::AnsiQuotes); +} + +bool MySQLParserBase::isTextStringLiteral() +{ + return !this->isSqlModeActive(SqlMode::AnsiQuotes); +} + +bool MySQLParserBase::isStoredRoutineBody() +{ + return serverVersion >= 80032 && supportMle; +} + +bool MySQLParserBase::isSelectStatementWithInto() +{ + return serverVersion >= 80024 && serverVersion < 80031; +} From bf5c91fd98437652b65b4aeeb3094b8c5fc343e6 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sat, 30 Nov 2024 15:10:55 -0500 Subject: [PATCH 09/29] Add JavaScript port (generated from TypeScript port). --- sql/mysql/Oracle/JavaScript/MySQLLexerBase.js | 390 ++++++++++++++++++ .../Oracle/JavaScript/MySQLParserBase.js | 40 ++ sql/mysql/Oracle/JavaScript/SqlMode.js | 11 + sql/mysql/Oracle/JavaScript/SqlModes.js | 36 ++ sql/mysql/Oracle/desc.xml | 2 +- 5 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 sql/mysql/Oracle/JavaScript/MySQLLexerBase.js create mode 100644 sql/mysql/Oracle/JavaScript/MySQLParserBase.js create mode 100644 sql/mysql/Oracle/JavaScript/SqlMode.js create mode 100644 sql/mysql/Oracle/JavaScript/SqlModes.js diff --git a/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js b/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js new file mode 100644 index 0000000000..8774e52bb6 --- /dev/null +++ b/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js @@ -0,0 +1,390 @@ +/* + * Copyright © 2024, Oracle and/or its affiliates + */ +var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) { + if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter"); + if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it"); + return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver); +}; +var _a, _MySQLLexerBase_longString, _MySQLLexerBase_longLength, _MySQLLexerBase_signedLongString, _MySQLLexerBase_longLongString, _MySQLLexerBase_longLongLength, _MySQLLexerBase_signedLongLongString, _MySQLLexerBase_signedLongLongLength, _MySQLLexerBase_unsignedLongLongString, _MySQLLexerBase_unsignedLongLongLength; +/* eslint-disable no-underscore-dangle */ +/* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ +import { Lexer, Token } from "antlr4"; +import { CommonToken } from "antlr4"; +import MySQLLexer from "./MySQLLexer.js"; +import SqlMode from "./SqlMode.js"; +import SqlModes from "./SqlModes.js"; +/** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ +class MySQLLexerBase extends Lexer { + constructor(input) { + super(input); + this.serverVersion = 0; + this.sqlModes = new Set(); + /** Enable Multi Language Extension support. */ + this.supportMle = true; + this.charSets = new Set(); // Used to check repertoires. + this.inVersionComment = false; + this.pendingTokens = []; + this.serverVersion = 80200; + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); + } + /** + * Determines if the given SQL mode is currently active in the lexer. + * + * @param mode The mode to check. + * + * @returns True if the mode is one of the currently active modes. + */ + isSqlModeActive(mode) { + return this.sqlModes.has(mode); + } + /** + * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. + */ + reset() { + this.inVersionComment = false; + super.reset(); + } + /** + * Implements the multi token feature required in our lexer. + * A lexer rule can emit more than a single token, if needed. + * + * @returns The next token in the token stream. + */ + nextToken() { + // First respond with pending tokens to the next token request, if there are any. + let pending = this.pendingTokens.shift(); + if (pending) { + return pending; + } + // Let the main lexer class run the next token recognition. + // This might create additional tokens again. + const next = super.nextToken(); + pending = this.pendingTokens.shift(); + if (pending) { + this.pendingTokens.push(next); + return pending; + } + return next; + } + /** + * Checks if the version number in the token text is less than or equal to the current server version. + * + * @param text The text from a matched token. + * @returns True if so the number matches, otherwise false. + */ + checkMySQLVersion(text) { + if (text.length < 8) { // Minimum is: /*!12345 + return false; + } + // Skip version comment introducer. + const version = parseInt(text.substring(3), 10); + if (version <= this.serverVersion) { + this.inVersionComment = true; + return true; + } + return false; + } + /** + * Called when a keyword was consumed that represents an internal MySQL function and checks if that keyword is + * followed by an open parenthesis. If not then it is not considered a keyword but treated like a normal identifier. + * + * @param proposed The token type to use if the check succeeds. + * + * @returns If a function call is found then return the proposed token type, otherwise just IDENTIFIER. + */ + determineFunction(proposed) { + // Skip any whitespace character if the sql mode says they should be ignored, + // before actually trying to match the open parenthesis. + let input = String.fromCharCode(this._input.LA(1)); + if (this.isSqlModeActive(SqlMode.IgnoreSpace)) { + while (input === " " || input === "\t" || input === "\r" || input === "\n") { + this._interp.consume(this._input); + // this.channel = Lexer.HIDDEN; + this._type = MySQLLexer.WHITESPACE; + input = String.fromCharCode(this._input.LA(1)); + } + } + return input === "(" ? proposed : MySQLLexer.IDENTIFIER; + } + /** + * Checks the given text and determines the smallest number type from it. Code has been taken from sql_lex.cc. + * + * @param text The text to parse (which must be a number). + * + * @returns The token type for that text. + */ + determineNumericType(text) { + // The original code checks for leading +/- but actually that can never happen, neither in the + // server parser (as a digit is used to trigger processing in the lexer) nor in our parser + // as our rules are defined without signs. But we do it anyway for maximum compatibility. + let length = text.length - 1; + if (length < __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLength)) { // quick normal case + return MySQLLexer.INT_NUMBER; + } + let negative = false; + let index = 0; + if (text.charAt(index) === "+") { // Remove sign and pre-zeros + ++index; + --length; + } + else if (text.charAt(index) === "-") { + ++index; + --length; + negative = true; + } + while (text.charAt(index) === "0" && length > 0) { + ++index; + --length; + } + if (length < __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLength)) { + return MySQLLexer.INT_NUMBER; + } + let smaller; + let bigger; + let cmp; + if (negative) { + if (length === __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLength)) { + cmp = __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_signedLongString).substring(1); + smaller = MySQLLexer.INT_NUMBER; // If <= signed_long_str + bigger = MySQLLexer.LONG_NUMBER; // If >= signed_long_str + } + else if (length < __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_signedLongLongLength)) { + return MySQLLexer.LONG_NUMBER; + } + else if (length > __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_signedLongLongLength)) { + return MySQLLexer.DECIMAL_NUMBER; + } + else { + cmp = __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_signedLongLongString).substring(1); + smaller = MySQLLexer.LONG_NUMBER; // If <= signed_longlong_str + bigger = MySQLLexer.DECIMAL_NUMBER; + } + } + else { + if (length === __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLength)) { + cmp = __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longString); + smaller = MySQLLexer.INT_NUMBER; + bigger = MySQLLexer.LONG_NUMBER; + } + else if (length < __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLongLength)) { + return MySQLLexer.LONG_NUMBER; + } + else if (length > __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLongLength)) { + if (length > __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_unsignedLongLongLength)) { + return MySQLLexer.DECIMAL_NUMBER; + } + cmp = __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_unsignedLongLongString); + smaller = MySQLLexer.ULONGLONG_NUMBER; + bigger = MySQLLexer.DECIMAL_NUMBER; + } + else { + cmp = __classPrivateFieldGet(_a, _a, "f", _MySQLLexerBase_longLongString); + smaller = MySQLLexer.LONG_NUMBER; + bigger = MySQLLexer.ULONGLONG_NUMBER; + } + } + let otherIndex = 0; + while (index < text.length && cmp.charAt(otherIndex++) === text.charAt(index++)) { + // + } + return text.charAt(index - 1) <= cmp.charAt(otherIndex - 1) ? smaller : bigger; + } + /** + * Checks if the given text corresponds to a charset defined in the server (text is preceded by an underscore). + * + * @param text The text to check. + * + * @returns UNDERSCORE_CHARSET if so, otherwise IDENTIFIER. + */ + checkCharset(text) { + return this.charSets.has(text) ? MySQLLexer.UNDERSCORE_CHARSET : MySQLLexer.IDENTIFIER; + } + /** + * Creates a DOT token in the token stream. + */ + emitDot() { + let t = new CommonToken([this, this._input], MySQLLexer.DOT_SYMBOL, 0, this._tokenStartCharIndex, this._tokenStartCharIndex); + this.pendingTokens.push(t); + ++this.column; + ++this._tokenStartCharIndex; + } + isServerVersionLt80024() { + return this.serverVersion < 80024; + } + isServerVersionGe80024() { + return this.serverVersion >= 80024; + } + isServerVersionGe80011() { + return this.serverVersion >= 80011; + } + isServerVersionGe80013() { + return this.serverVersion >= 80013; + } + isServerVersionLt80014() { + return this.serverVersion < 80014; + } + isServerVersionGe80014() { + return this.serverVersion >= 80014; + } + isServerVersionGe80017() { + return this.serverVersion >= 80017; + } + isServerVersionGe80018() { return this.serverVersion >= 80018; } + isMasterCompressionAlgorithm() { return this.serverVersion >= 80018 && this.isServerVersionLt80024(); } + isServerVersionLt80031() { + return this.serverVersion < 80031; + } + doLogicalOr() { + this._type = this.isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; + } + doIntNumber() { + this._type = this.determineNumericType(this.text); + } + doAdddate() { + this._type = this.determineFunction(MySQLLexer.ADDDATE_SYMBOL); + } + doBitAnd() { + this._type = this.determineFunction(MySQLLexer.BIT_AND_SYMBOL); + } + doBitOr() { + this._type = this.determineFunction(MySQLLexer.BIT_OR_SYMBOL); + } + doBitXor() { + this._type = this.determineFunction(MySQLLexer.BIT_XOR_SYMBOL); + } + doCast() { + this._type = this.determineFunction(MySQLLexer.CAST_SYMBOL); + } + doCount() { + this._type = this.determineFunction(MySQLLexer.COUNT_SYMBOL); + } + doCurdate() { + this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + } + doCurrentDate() { + this._type = this.determineFunction(MySQLLexer.CURDATE_SYMBOL); + } + doCurrentTime() { + this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + } + doCurtime() { + this._type = this.determineFunction(MySQLLexer.CURTIME_SYMBOL); + } + doDateAdd() { + this._type = this.determineFunction(MySQLLexer.DATE_ADD_SYMBOL); + } + doDateSub() { + this._type = this.determineFunction(MySQLLexer.DATE_SUB_SYMBOL); + } + doExtract() { + this._type = this.determineFunction(MySQLLexer.EXTRACT_SYMBOL); + } + doGroupConcat() { + this._type = this.determineFunction(MySQLLexer.GROUP_CONCAT_SYMBOL); + } + doMax() { + this._type = this.determineFunction(MySQLLexer.MAX_SYMBOL); + } + doMid() { + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + } + doMin() { + this._type = this.determineFunction(MySQLLexer.MIN_SYMBOL); + } + doNot() { + this._type = this.isSqlModeActive(SqlMode.HighNotPrecedence) ? MySQLLexer.NOT2_SYMBOL : MySQLLexer.NOT_SYMBOL; + } + doNow() { + this._type = this.determineFunction(MySQLLexer.NOW_SYMBOL); + } + doPosition() { + this._type = this.determineFunction(MySQLLexer.POSITION_SYMBOL); + } + doSessionUser() { + this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); + } + doStddevSamp() { + this._type = this.determineFunction(MySQLLexer.STDDEV_SAMP_SYMBOL); + } + doStddev() { + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + } + doStddevPop() { + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + } + doStd() { + this._type = this.determineFunction(MySQLLexer.STD_SYMBOL); + } + doSubdate() { + this._type = this.determineFunction(MySQLLexer.SUBDATE_SYMBOL); + } + doSubstr() { + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + } + doSubstring() { + this._type = this.determineFunction(MySQLLexer.SUBSTRING_SYMBOL); + } + doSum() { + this._type = this.determineFunction(MySQLLexer.SUM_SYMBOL); + } + doSysdate() { + this._type = this.determineFunction(MySQLLexer.SYSDATE_SYMBOL); + } + doSystemUser() { + this._type = this.determineFunction(MySQLLexer.USER_SYMBOL); + } + doTrim() { + this._type = this.determineFunction(MySQLLexer.TRIM_SYMBOL); + } + doVariance() { + this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + } + doVarPop() { + this._type = this.determineFunction(MySQLLexer.VARIANCE_SYMBOL); + } + doVarSamp() { + this._type = this.determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); + } + doUnderscoreCharset() { + this._type = this.checkCharset(this.text); + } + isVersionComment() { + return this.checkMySQLVersion(this.text); + } + isBackTickQuotedId() { + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + } + isDoubleQuotedText() { + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + } + isSingleQuotedText() { + return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); + } + emit() { + let t = super.emit(); + if (t.type == MySQLLexer.WHITESPACE) + t.channel = Token.HIDDEN_CHANNEL; + return t; + } + startInVersionComment() { + this.inVersionComment = true; + } + endInVersionComment() { + this.inVersionComment = false; + } + isInVersionComment() { + return this.inVersionComment; + } +} +_a = MySQLLexerBase; +_MySQLLexerBase_longString = { value: "2147483647" }; +_MySQLLexerBase_longLength = { value: 10 }; +_MySQLLexerBase_signedLongString = { value: "-2147483648" }; +_MySQLLexerBase_longLongString = { value: "9223372036854775807" }; +_MySQLLexerBase_longLongLength = { value: 19 }; +_MySQLLexerBase_signedLongLongString = { value: "-9223372036854775808" }; +_MySQLLexerBase_signedLongLongLength = { value: 19 }; +_MySQLLexerBase_unsignedLongLongString = { value: "18446744073709551615" }; +_MySQLLexerBase_unsignedLongLongLength = { value: 20 }; +export default MySQLLexerBase; diff --git a/sql/mysql/Oracle/JavaScript/MySQLParserBase.js b/sql/mysql/Oracle/JavaScript/MySQLParserBase.js new file mode 100644 index 0000000000..9098a5b5d7 --- /dev/null +++ b/sql/mysql/Oracle/JavaScript/MySQLParserBase.js @@ -0,0 +1,40 @@ +/* + * Copyright © 2024, Oracle and/or its affiliates + */ +import { Parser } from "antlr4"; +import SqlMode from "./SqlMode.js"; +import SqlModes from "./SqlModes.js"; +export default class MySQLParserBase extends Parser { + constructor(input) { + super(input); + // To parameterize the parsing process. + this.serverVersion = 0; + this.sqlModes = new Set(); + /** Enable Multi Language Extension support. */ + this.supportMle = true; + this.serverVersion = 80200; + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); + } + /** + * Determines if the given SQL mode is currently active in the lexer. + * + * @param mode The mode to check. + * + * @returns True if the mode is one of the currently active modes. + */ + isSqlModeActive(mode) { + return this.sqlModes.has(mode); + } + isPureIdentifier() { + return this.isSqlModeActive(SqlMode.AnsiQuotes); + } + isTextStringLiteral() { + return !this.isSqlModeActive(SqlMode.AnsiQuotes); + } + isStoredRoutineBody() { + return this.serverVersion >= 80032 && this.supportMle; + } + isSelectStatementWithInto() { + return this.serverVersion >= 80024 && this.serverVersion < 80031; + } +} diff --git a/sql/mysql/Oracle/JavaScript/SqlMode.js b/sql/mysql/Oracle/JavaScript/SqlMode.js new file mode 100644 index 0000000000..0786db3260 --- /dev/null +++ b/sql/mysql/Oracle/JavaScript/SqlMode.js @@ -0,0 +1,11 @@ +/** SQL modes that control parsing behavior. */ +var SqlMode; +(function (SqlMode) { + SqlMode[SqlMode["NoMode"] = 0] = "NoMode"; + SqlMode[SqlMode["AnsiQuotes"] = 1] = "AnsiQuotes"; + SqlMode[SqlMode["HighNotPrecedence"] = 2] = "HighNotPrecedence"; + SqlMode[SqlMode["PipesAsConcat"] = 3] = "PipesAsConcat"; + SqlMode[SqlMode["IgnoreSpace"] = 4] = "IgnoreSpace"; + SqlMode[SqlMode["NoBackslashEscapes"] = 5] = "NoBackslashEscapes"; +})(SqlMode || (SqlMode = {})); +export default SqlMode; diff --git a/sql/mysql/Oracle/JavaScript/SqlModes.js b/sql/mysql/Oracle/JavaScript/SqlModes.js new file mode 100644 index 0000000000..b21a70b3ae --- /dev/null +++ b/sql/mysql/Oracle/JavaScript/SqlModes.js @@ -0,0 +1,36 @@ +/** SQL modes that control parsing behavior. */ +import SqlMode from "./SqlMode.js"; +export class SqlModes { + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + static sqlModeFromString(modes) { + var result = new Set(); + const parts = modes.toUpperCase().split(","); + parts.forEach((mode) => { + if (mode === "ANSI" || mode === "DB2" || mode === "MAXDB" || mode === "MSSQL" || mode === "ORACLE" || + mode === "POSTGRESQL") { + result.add(SqlMode.AnsiQuotes).add(SqlMode.PipesAsConcat).add(SqlMode.IgnoreSpace); + } + else if (mode === "ANSI_QUOTES") { + result.add(SqlMode.AnsiQuotes); + } + else if (mode === "PIPES_AS_CONCAT") { + result.add(SqlMode.PipesAsConcat); + } + else if (mode === "NO_BACKSLASH_ESCAPES") { + result.add(SqlMode.NoBackslashEscapes); + } + else if (mode === "IGNORE_SPACE") { + result.add(SqlMode.IgnoreSpace); + } + else if (mode === "HIGH_NOT_PRECEDENCE" || mode === "MYSQL323" || mode === "MYSQL40") { + result.add(SqlMode.HighNotPrecedence); + } + }); + return result; + } +} +export default SqlModes; diff --git a/sql/mysql/Oracle/desc.xml b/sql/mysql/Oracle/desc.xml index 7dbffd7764..b6341320c0 100644 --- a/sql/mysql/Oracle/desc.xml +++ b/sql/mysql/Oracle/desc.xml @@ -1,5 +1,5 @@ - Antlr4ng;Cpp;CSharp;Dart;Java;Python3;TypeScript + Antlr4ng;Cpp;CSharp;Dart;Java;JavaScript;Python3;TypeScript examples/**/*.sql From 32f4c2821f298c920a7d59ad27bce119d55946d6 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sun, 1 Dec 2024 04:16:29 -0500 Subject: [PATCH 10/29] Adding Go port (incomplete). --- sql/mysql/Oracle/Go/MySQLLexerBase.go | 12 ++++++ sql/mysql/Oracle/Go/MySQLParserBase.go | 9 ++++ sql/mysql/Oracle/Go/Set.go | 46 ++++++++++++++++++++ sql/mysql/Oracle/Go/SqlMode.go | 27 ++++++++++++ sql/mysql/Oracle/Go/transformGrammar.py | 56 +++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 sql/mysql/Oracle/Go/MySQLLexerBase.go create mode 100644 sql/mysql/Oracle/Go/MySQLParserBase.go create mode 100644 sql/mysql/Oracle/Go/Set.go create mode 100644 sql/mysql/Oracle/Go/SqlMode.go create mode 100644 sql/mysql/Oracle/Go/transformGrammar.py diff --git a/sql/mysql/Oracle/Go/MySQLLexerBase.go b/sql/mysql/Oracle/Go/MySQLLexerBase.go new file mode 100644 index 0000000000..9698959006 --- /dev/null +++ b/sql/mysql/Oracle/Go/MySQLLexerBase.go @@ -0,0 +1,12 @@ +package parser + +import ( + "regexp" + "github.com/antlr4-go/antlr/v4" +) + +type MySQLLexerBase struct { + *antlr.BaseLexer + serverVersion int +} + diff --git a/sql/mysql/Oracle/Go/MySQLParserBase.go b/sql/mysql/Oracle/Go/MySQLParserBase.go new file mode 100644 index 0000000000..d7d6fc7092 --- /dev/null +++ b/sql/mysql/Oracle/Go/MySQLParserBase.go @@ -0,0 +1,9 @@ +package parser + +import ( + "github.com/antlr4-go/antlr/v4" +) + +type MySQLParserBase struct { + *antlr.BaseParser +} diff --git a/sql/mysql/Oracle/Go/Set.go b/sql/mysql/Oracle/Go/Set.go new file mode 100644 index 0000000000..8945f56630 --- /dev/null +++ b/sql/mysql/Oracle/Go/Set.go @@ -0,0 +1,46 @@ +package parser + +// Set is a type that represents a set implemented as a map +type Set[T comparable] struct { + items map[T]struct{} +} + +// NewSet creates and returns a new set +func NewSet[T comparable]() *Set[T] { + return &Set[T]{items: make(map[T]struct{})} +} + +// Add adds an element to the set +func (s *Set[T]) Add(element T) { + s.items[element] = struct{}{} +} + +// Remove removes an element from the set +func (s *Set[T]) Remove(element T) { + delete(s.items, element) +} + +// Contains checks if an element is in the set +func (s *Set[T]) Contains(element T) bool { + _, exists := s.items[element] + return exists +} + +// Size returns the number of elements in the set +func (s *Set[T]) Size() int { + return len(s.items) +} + +// Clear removes all elements from the set +func (s *Set[T]) Clear() { + s.items = make(map[T]struct{}) +} + +// Elements returns all elements in the set as a slice +func (s *Set[T]) Elements() []T { + elements := make([]T, 0, len(s.items)) + for key := range s.items { + elements = append(elements, key) + } + return elements +} diff --git a/sql/mysql/Oracle/Go/SqlMode.go b/sql/mysql/Oracle/Go/SqlMode.go new file mode 100644 index 0000000000..ae9a4c65a2 --- /dev/null +++ b/sql/mysql/Oracle/Go/SqlMode.go @@ -0,0 +1,27 @@ +package parser + +// SqlMode represents SQL modes that control parsing behavior. + +type SqlMode int + +// Enum values for SqlMode +const ( + NoMode SqlMode = iota + AnsiQuotes + HighNotPrecedence + PipesAsConcat + IgnoreSpace + NoBackslashEscapes +) + +// String provides string representation for SqlMode +func (sm SqlMode) String() string { + return [...]string{ + "NoMode", + "AnsiQuotes", + "HighNotPrecedence", + "PipesAsConcat", + "IgnoreSpace", + "NoBackslashEscapes", + }[sm] +} diff --git a/sql/mysql/Oracle/Go/transformGrammar.py b/sql/mysql/Oracle/Go/transformGrammar.py new file mode 100644 index 0000000000..e33021d1f7 --- /dev/null +++ b/sql/mysql/Oracle/Go/transformGrammar.py @@ -0,0 +1,56 @@ +import sys, os, re, shutil +from glob import glob +from pathlib import Path + +def main(argv): + for file in glob("./parser/*Lexer.g4"): + fix_lexer(file) + for file in glob("./parser/*Parser.g4"): + fix_parser(file) + +def fix_lexer(file_path): + print("Altering " + file_path) + if not os.path.exists(file_path): + print(f"Could not find file: {file_path}") + sys.exit(1) + parts = os.path.split(file_path) + file_name = parts[-1] + + shutil.move(file_path, file_path + ".bak") + input_file = open(file_path + ".bak",'r') + output_file = open(file_path, 'w') + for x in input_file: + if 'this.' in x and '}?' in x: + x = x.replace('this.', 'p.') + elif 'this.' in x: + x = x.replace('this.', 'l.') + output_file.write(x) + output_file.flush() + + print("Writing ...") + input_file.close() + output_file.close() + +def fix_parser(file_path): + print("Altering " + file_path) + if not os.path.exists(file_path): + print(f"Could not find file: {file_path}") + sys.exit(1) + parts = os.path.split(file_path) + file_name = parts[-1] + + shutil.move(file_path, file_path + ".bak") + input_file = open(file_path + ".bak",'r') + output_file = open(file_path, 'w') + for x in input_file: + if 'this.' in x: + x = x.replace('this.', 'p.') + output_file.write(x) + output_file.flush() + + print("Writing ...") + input_file.close() + output_file.close() + +if __name__ == '__main__': + main(sys.argv) From 102dd6586d193c4055a33b79c34f7d6d05867d65 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sun, 1 Dec 2024 05:19:07 -0500 Subject: [PATCH 11/29] Clean up. --- sql/mysql/Oracle/CSharp/MySQLLexerBase.cs | 49 ++-------------------- sql/mysql/Oracle/CSharp/MySQLParserBase.cs | 33 +-------------- sql/mysql/Oracle/CSharp/SqlMode.cs | 9 ++++ sql/mysql/Oracle/CSharp/SqlModes.cs | 46 ++++++++++++++++++++ 4 files changed, 59 insertions(+), 78 deletions(-) create mode 100644 sql/mysql/Oracle/CSharp/SqlMode.cs create mode 100644 sql/mysql/Oracle/CSharp/SqlModes.cs diff --git a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs index ecd61ea3ae..f2524e14e2 100644 --- a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs @@ -5,22 +5,10 @@ /* eslint-disable no-underscore-dangle */ /* cspell: ignore antlr, longlong, ULONGLONG, MAXDB */ +using Antlr4.Runtime; using System; using System.Collections.Generic; using System.IO; -using System.Threading.Channels; -using Antlr4.Runtime; -using static System.Net.WebRequestMethods; - -/** SQL modes that control parsing behavior. */ -public enum SqlMode { - NoMode, - AnsiQuotes, - HighNotPrecedence, - PipesAsConcat, - IgnoreSpace, - NoBackslashEscapes, -}; /** The base lexer class provides a number of functions needed in actions in the lexer (grammar). */ public class MySQLLexerBase : Lexer { @@ -58,14 +46,14 @@ protected MySQLLexerBase(ICharStream input, TextWriter output, TextWriter errorO : base(input, output, errorOutput) { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } public MySQLLexerBase(ICharStream input) : base(input) { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** @@ -80,37 +68,6 @@ public bool isSqlModeActive(SqlMode mode) return this.sqlModes.Contains(mode); } - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public void sqlModeFromString(string modes) - { - this.sqlModes = new HashSet(); - - var parts = modes.ToUpper().Split(","); - foreach (var mode in parts) - { - if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || mode == "ORACLE" || - mode == "POSTGRESQL") { - this.sqlModes.Add(SqlMode.AnsiQuotes); - this.sqlModes.Add(SqlMode.PipesAsConcat); - this.sqlModes.Add(SqlMode.IgnoreSpace); - } else if (mode == "ANSI_QUOTES") { - this.sqlModes.Add(SqlMode.AnsiQuotes); - } else if (mode == "PIPES_AS_CONCAT") { - this.sqlModes.Add(SqlMode.PipesAsConcat); - } else if (mode == "NO_BACKSLASH_ESCAPES") { - this.sqlModes.Add(SqlMode.NoBackslashEscapes); - } else if (mode == "IGNORE_SPACE") { - this.sqlModes.Add(SqlMode.IgnoreSpace); - } else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40") { - this.sqlModes.Add(SqlMode.HighNotPrecedence); - } - } - } - /** * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. */ diff --git a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs index 692d701c88..b7fc9114e7 100644 --- a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs @@ -18,7 +18,7 @@ public abstract class MySQLParserBase : Parser { protected MySQLParserBase(ITokenStream input, TextWriter output, TextWriter errorOutput) : base(input, output, errorOutput) { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } /** @@ -51,35 +51,4 @@ public bool isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } - - /** - * Converts a mode string into individual mode flags. - * - * @param modes The input string to parse. - */ - public void sqlModeFromString(string modes) - { - this.sqlModes = new HashSet(); - - var parts = modes.ToUpper().Split(","); - foreach (var mode in parts) - { - if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || mode == "ORACLE" || - mode == "POSTGRESQL") { - this.sqlModes.Add(SqlMode.AnsiQuotes); - this.sqlModes.Add(SqlMode.PipesAsConcat); - this.sqlModes.Add(SqlMode.IgnoreSpace); - } else if (mode == "ANSI_QUOTES") { - this.sqlModes.Add(SqlMode.AnsiQuotes); - } else if (mode == "PIPES_AS_CONCAT") { - this.sqlModes.Add(SqlMode.PipesAsConcat); - } else if (mode == "NO_BACKSLASH_ESCAPES") { - this.sqlModes.Add(SqlMode.NoBackslashEscapes); - } else if (mode == "IGNORE_SPACE") { - this.sqlModes.Add(SqlMode.IgnoreSpace); - } else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40") { - this.sqlModes.Add(SqlMode.HighNotPrecedence); - } - } - } } diff --git a/sql/mysql/Oracle/CSharp/SqlMode.cs b/sql/mysql/Oracle/CSharp/SqlMode.cs new file mode 100644 index 0000000000..b1e4e0ab8a --- /dev/null +++ b/sql/mysql/Oracle/CSharp/SqlMode.cs @@ -0,0 +1,9 @@ +/** SQL modes that control parsing behavior. */ +public enum SqlMode { + NoMode, + AnsiQuotes, + HighNotPrecedence, + PipesAsConcat, + IgnoreSpace, + NoBackslashEscapes, +} diff --git a/sql/mysql/Oracle/CSharp/SqlModes.cs b/sql/mysql/Oracle/CSharp/SqlModes.cs new file mode 100644 index 0000000000..678aa411ed --- /dev/null +++ b/sql/mysql/Oracle/CSharp/SqlModes.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; + +public class SqlModes +{ + /** + * Converts a mode string into individual mode flags. + * + * @param modes The input string to parse. + */ + public static HashSet sqlModeFromString(string modes) + { + var result = new HashSet(); + var parts = modes.ToUpper().Split(","); + foreach (var mode in parts) + { + if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || mode == "ORACLE" || + mode == "POSTGRESQL") + { + result.Add(SqlMode.AnsiQuotes); + result.Add(SqlMode.PipesAsConcat); + result.Add(SqlMode.IgnoreSpace); + } + else if (mode == "ANSI_QUOTES") + { + result.Add(SqlMode.AnsiQuotes); + } + else if (mode == "PIPES_AS_CONCAT") + { + result.Add(SqlMode.PipesAsConcat); + } + else if (mode == "NO_BACKSLASH_ESCAPES") + { + result.Add(SqlMode.NoBackslashEscapes); + } + else if (mode == "IGNORE_SPACE") + { + result.Add(SqlMode.IgnoreSpace); + } + else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40") + { + result.Add(SqlMode.HighNotPrecedence); + } + } + return result; + } +} From 2cfb87abcfad26497adf578911e0313e20b1b915 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Sun, 1 Dec 2024 06:13:34 -0500 Subject: [PATCH 12/29] Clean up Dart port. --- sql/mysql/Oracle/Dart/MySQLLexerBase.dart | 119 ++++++++------------- sql/mysql/Oracle/Dart/MySQLParserBase.dart | 45 ++------ sql/mysql/Oracle/Dart/SqlMode.dart | 9 ++ sql/mysql/Oracle/Dart/SqlModes.dart | 38 +++++++ 4 files changed, 96 insertions(+), 115 deletions(-) create mode 100644 sql/mysql/Oracle/Dart/SqlMode.dart create mode 100644 sql/mysql/Oracle/Dart/SqlModes.dart diff --git a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart index f78a3e402c..93d37abcb3 100644 --- a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart +++ b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart @@ -5,24 +5,16 @@ import 'dart:convert'; import 'dart:collection'; import 'MySQLLexer.dart'; import 'MySQLParser.dart'; - -/** SQL modes that control parsing behavior. */ -enum SqlMode { - noMode, - ansiQuotes, - highNotPrecedence, - pipesAsConcat, - ignoreSpace, - noBackslashEscapes, -} +import 'SqlMode.dart'; +import 'SqlModes.dart'; /** Base lexer class providing functions needed in actions. */ abstract class MySQLLexerBase extends Lexer { int serverVersion = 0; - Set sqlModes = {}; + HashSet sqlModes = HashSet(); bool supportMle = true; - Set charSets = {}; + HashSet charSets = HashSet(); bool inVersionComment = false; Queue pendingTokens = Queue(); @@ -41,38 +33,13 @@ abstract class MySQLLexerBase extends Lexer MySQLLexerBase(CharStream input) : super(input) { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } bool isSqlModeActive(SqlMode mode) { return sqlModes.contains(mode); } - void sqlModeFromString(String modes) { - sqlModes.clear(); - List parts = modes.toUpperCase().split(','); - - for (String mode in parts) { - if (['ANSI', 'DB2', 'MAXDB', 'MSSQL', 'ORACLE', 'POSTGRESQL'] - .contains(mode)) { - sqlModes.add(SqlMode.ansiQuotes); - sqlModes.add(SqlMode.pipesAsConcat); - sqlModes.add(SqlMode.ignoreSpace); - } else if (mode == 'ANSI_QUOTES') { - sqlModes.add(SqlMode.ansiQuotes); - } else if (mode == 'PIPES_AS_CONCAT') { - sqlModes.add(SqlMode.pipesAsConcat); - } else if (mode == 'NO_BACKSLASH_ESCAPES') { - sqlModes.add(SqlMode.noBackslashEscapes); - } else if (mode == 'IGNORE_SPACE') { - sqlModes.add(SqlMode.ignoreSpace); - } else if (['HIGH_NOT_PRECEDENCE', 'MYSQL323', 'MYSQL40'] - .contains(mode)) { - sqlModes.add(SqlMode.highNotPrecedence); - } - } - } - /** * Resets the lexer by setting initial values to transient member, resetting the input stream position etc. */ @@ -115,19 +82,19 @@ abstract class MySQLLexerBase extends Lexer * @param text The text from a matched token. * @returns True if so the number matches, otherwise false. */ - bool checkMySQLVersion(String text) { - if (text.length < 8) { - return false; - } + bool checkMySQLVersion(String text) { + if (text.length < 8) { + return false; + } - int version = int.parse(text.substring(3)); - if (version <= serverVersion) { - inVersionComment = true; - return true; - } + int version = int.parse(text.substring(3)); + if (version <= serverVersion) { + inVersionComment = true; + return true; + } - return false; - } + return false; + } /** * Called when a keyword was consumed that represents an internal MySQL function and checks if that keyword is @@ -137,21 +104,21 @@ abstract class MySQLLexerBase extends Lexer * * @returns If a function call is found then return the proposed token type, otherwise just IDENTIFIER. */ - int determineFunction(int proposed) { + int determineFunction(int proposed) { var input = this.inputStream.LA(1) ?? 0; - if (isSqlModeActive(SqlMode.ignoreSpace)) { - while ([' ', '\t', '\r', '\n'].contains(String.fromCharCode(input))) { - // Consume logic based on InputStream equivalent. + if (isSqlModeActive(SqlMode.ignoreSpace)) { + while ([' ', '\t', '\r', '\n'].contains(String.fromCharCode(input))) { + // Consume logic based on InputStream equivalent. this.interpreter?.consume(this.inputStream); this.channel = Lexer.HIDDEN; this.type = MySQLLexer.TOKEN_WHITESPACE; - var c = this.inputStream.LA(1); + var c = this.inputStream.LA(1); input = c ?? 0; - } - } - var r = input == '('.codeUnitAt(0) ? proposed : MySQLLexer.TOKEN_IDENTIFIER; - return r; - } + } + } + var r = input == '('.codeUnitAt(0) ? proposed : MySQLLexer.TOKEN_IDENTIFIER; + return r; + } /** * Checks the given text and determines the smallest number type from it. Code has been taken from sql_lex.cc. @@ -233,7 +200,7 @@ abstract class MySQLLexerBase extends Lexer // } - var i = text[index - 1].compareTo(cmp[otherIndex - 1]); + var i = text[index - 1].compareTo(cmp[otherIndex - 1]); return i <= 0 ? smaller : bigger; } @@ -246,9 +213,9 @@ abstract class MySQLLexerBase extends Lexer */ int checkCharset(String text) { - var z = this.charSets.contains(text); + var z = this.charSets.contains(text); var r = z ? MySQLLexer.TOKEN_UNDERSCORE_CHARSET : MySQLLexer.TOKEN_IDENTIFIER; - return r; + return r; } /** @@ -258,14 +225,14 @@ abstract class MySQLLexerBase extends Lexer { var ctf = this.tokenFactory; Token t = ctf.create( - MySQLLexer.TOKEN_DOT_SYMBOL, - this.text, - Pair(this, this.inputStream), + MySQLLexer.TOKEN_DOT_SYMBOL, + this.text, + Pair(this, this.inputStream), this.channel, - this.tokenStartCharIndex, - this.tokenStartCharIndex, - this.line, - this.charPositionInLine); + this.tokenStartCharIndex, + this.tokenStartCharIndex, + this.line, + this.charPositionInLine); this.pendingTokens.add(t); ++this.charPositionInLine; ++this.tokenStartCharIndex; @@ -356,14 +323,14 @@ abstract class MySQLLexerBase extends Lexer { var ctf = this.tokenFactory; Token t = ctf.create( - this.type, - (this.text!=null?(this.justEmitedDot?this.text.substring(1):this.text):null), - Pair(this, this.inputStream), + this.type, + (this.text!=null?(this.justEmitedDot?this.text.substring(1):this.text):null), + Pair(this, this.inputStream), this.channel, - this.tokenStartCharIndex + (this.justEmitedDot?1:0), - CharIndex - 1, - this.line, - this.charPositionInLine); + this.tokenStartCharIndex + (this.justEmitedDot?1:0), + CharIndex - 1, + this.line, + this.charPositionInLine); this.justEmitedDot = false; super.emit(t); return t; diff --git a/sql/mysql/Oracle/Dart/MySQLParserBase.dart b/sql/mysql/Oracle/Dart/MySQLParserBase.dart index 47f1c61b8f..d9749f002e 100644 --- a/sql/mysql/Oracle/Dart/MySQLParserBase.dart +++ b/sql/mysql/Oracle/Dart/MySQLParserBase.dart @@ -5,30 +5,22 @@ import 'dart:convert'; import 'dart:collection'; import 'MySQLLexer.dart'; import 'MySQLParser.dart'; - -/** SQL modes that control parsing behavior. */ -enum SqlMode { - noMode, - ansiQuotes, - highNotPrecedence, - pipesAsConcat, - ignoreSpace, - noBackslashEscapes, -} +import 'SqlMode.dart'; +import 'SqlModes.dart'; /** Base parser class for MySQL parsing. */ abstract class MySQLParserBase extends Parser { // To parameterize the parsing process. int serverVersion = 0; - Set sqlModes = {}; + HashSet sqlModes = HashSet(); bool supportMle = true; MySQLParserBase(TokenStream input) : super(input) - { + { this.serverVersion = 80200; - this.sqlModeFromString("ANSI_QUOTES"); - } + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); + } /// Determines if the given SQL mode is currently active. bool isSqlModeActive(SqlMode mode) { @@ -54,29 +46,4 @@ abstract class MySQLParserBase extends Parser { bool isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } - - void sqlModeFromString(String modes) { - sqlModes.clear(); - List parts = modes.toUpperCase().split(','); - - for (String mode in parts) { - if (['ANSI', 'DB2', 'MAXDB', 'MSSQL', 'ORACLE', 'POSTGRESQL'] - .contains(mode)) { - sqlModes.add(SqlMode.ansiQuotes); - sqlModes.add(SqlMode.pipesAsConcat); - sqlModes.add(SqlMode.ignoreSpace); - } else if (mode == 'ANSI_QUOTES') { - sqlModes.add(SqlMode.ansiQuotes); - } else if (mode == 'PIPES_AS_CONCAT') { - sqlModes.add(SqlMode.pipesAsConcat); - } else if (mode == 'NO_BACKSLASH_ESCAPES') { - sqlModes.add(SqlMode.noBackslashEscapes); - } else if (mode == 'IGNORE_SPACE') { - sqlModes.add(SqlMode.ignoreSpace); - } else if (['HIGH_NOT_PRECEDENCE', 'MYSQL323', 'MYSQL40'] - .contains(mode)) { - sqlModes.add(SqlMode.highNotPrecedence); - } - } - } } diff --git a/sql/mysql/Oracle/Dart/SqlMode.dart b/sql/mysql/Oracle/Dart/SqlMode.dart new file mode 100644 index 0000000000..275dba6294 --- /dev/null +++ b/sql/mysql/Oracle/Dart/SqlMode.dart @@ -0,0 +1,9 @@ +/** SQL modes that control parsing behavior. */ +enum SqlMode { + noMode, + ansiQuotes, + highNotPrecedence, + pipesAsConcat, + ignoreSpace, + noBackslashEscapes, +} diff --git a/sql/mysql/Oracle/Dart/SqlModes.dart b/sql/mysql/Oracle/Dart/SqlModes.dart new file mode 100644 index 0000000000..71859cd636 --- /dev/null +++ b/sql/mysql/Oracle/Dart/SqlModes.dart @@ -0,0 +1,38 @@ +import 'package:antlr4/antlr4.dart'; +import 'dart:io'; +import 'dart:core'; +import 'dart:convert'; +import 'dart:collection'; +import 'MySQLLexer.dart'; +import 'MySQLParser.dart'; +import 'SqlMode.dart'; + +class SqlModes +{ + static HashSet sqlModeFromString(String modes) + { + var result = HashSet(); + List parts = modes.toUpperCase().split(','); + for (String mode in parts) + { + if (['ANSI', 'DB2', 'MAXDB', 'MSSQL', 'ORACLE', 'POSTGRESQL'].contains(mode)) + { + result.add(SqlMode.ansiQuotes); + result.add(SqlMode.pipesAsConcat); + result.add(SqlMode.ignoreSpace); + } else if (mode == 'ANSI_QUOTES') { + result.add(SqlMode.ansiQuotes); + } else if (mode == 'PIPES_AS_CONCAT') { + result.add(SqlMode.pipesAsConcat); + } else if (mode == 'NO_BACKSLASH_ESCAPES') { + result.add(SqlMode.noBackslashEscapes); + } else if (mode == 'IGNORE_SPACE') { + result.add(SqlMode.ignoreSpace); + } else if (['HIGH_NOT_PRECEDENCE', 'MYSQL323', 'MYSQL40'] + .contains(mode)) { + result.add(SqlMode.highNotPrecedence); + } + } + return result; + } +} From e9d22d205fab75d045aa727222fe19c901cf1ad9 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 07:50:56 -0500 Subject: [PATCH 13/29] Split actions to different lines so transformGrammar.py doesn't need rewrite. Fix Go target (not working yet). --- sql/mysql/Oracle/Go/MySQLLexerBase.go | 302 ++++++++++++++++++++++++- sql/mysql/Oracle/Go/MySQLParserBase.go | 40 +++- sql/mysql/Oracle/Go/Set.go | 46 ---- sql/mysql/Oracle/MySQLLexer.g4 | 3 +- 4 files changed, 339 insertions(+), 52 deletions(-) delete mode 100644 sql/mysql/Oracle/Go/Set.go diff --git a/sql/mysql/Oracle/Go/MySQLLexerBase.go b/sql/mysql/Oracle/Go/MySQLLexerBase.go index 9698959006..4f7057cb62 100644 --- a/sql/mysql/Oracle/Go/MySQLLexerBase.go +++ b/sql/mysql/Oracle/Go/MySQLLexerBase.go @@ -1,12 +1,306 @@ package parser import ( - "regexp" - "github.com/antlr4-go/antlr/v4" + "strconv" + "github.com/antlr4-go/antlr/v4" ) type MySQLLexerBase struct { - *antlr.BaseLexer - serverVersion int + *antlr.BaseLexer + serverVersion int + sqlModes map[SqlMode]bool + supportMle bool + charSets map[string]bool + inVersionComment bool + pendingTokens []antlr.Token + justEmittedDot bool + longString string + longLength int + signedLongString string + longLongString string + longLongLength int + signedLongLongString string + signedLongLongLength int + unsignedLongLongString string + unsignedLongLongLength int +} + +var StaticMySQLLexerBase MySQLLexerBase + +func init() { + StaticMySQLLexerBase = MySQLLexerBase { + serverVersion: 80200, + supportMle: true, + charSets: make(map[string]bool), + inVersionComment: false, + longString: "2147483647", + longLength: 10, + signedLongString: "-2147483648", + longLongString: "9223372036854775807", + longLongLength: 19, + signedLongLongString: "-9223372036854775808", + signedLongLongLength: 19, + unsignedLongLongString: "18446744073709551615", + unsignedLongLongLength: 20, + } +} + +func NewMySQLLexerBase(input antlr.CharStream) *MySQLLexerBase { + r := &MySQLLexerBase{ + serverVersion: 80200, + supportMle: true, + charSets: make(map[string]bool), + inVersionComment: false, + longString: "2147483647", + longLength: 10, + signedLongString: "-2147483648", + longLongString: "9223372036854775807", + longLongLength: 19, + signedLongLongString: "-9223372036854775808", + signedLongLongLength: 19, + unsignedLongLongString: "18446744073709551615", + unsignedLongLongLength: 20, + } + r.sqlModes = make(map[SqlMode]bool) + return r +} + +func (l *MySQLLexerBase) MakeCommonToken(ttype int, text string) antlr.Token { + stop := l.TokenStartCharIndex - 1 + start := stop + if len(text) != 0 { + start = stop - len(text) + 1 + } + ctf := l.GetTokenFactory() + t := ctf.Create( + l.GetTokenSourceCharStreamPair(), + ttype, + text, + antlr.TokenDefaultChannel, + start, + l.TokenStartCharIndex-1, + l.TokenStartLine, + l.TokenStartColumn) + return t +} + +func (m *MySQLLexerBase) emitDot() { + m.pendingTokens = append(m.pendingTokens, m.MakeCommonToken(MySQLLexerDOT_SYMBOL, m.GetText())) +// ++this.Column; +// m.justEmitedDot = true +} + +func (m *MySQLLexerBase) isServerVersionLt80024() bool { return m.serverVersion < 80024 } +func (m *MySQLLexerBase) isServerVersionGe80024() bool { return m.serverVersion >= 80024 } +func (m *MySQLLexerBase) isServerVersionGe80011() bool { return m.serverVersion >= 80011 } +func (m *MySQLLexerBase) isServerVersionGe80013() bool { return m.serverVersion >= 80013 } +func (m *MySQLLexerBase) isServerVersionLt80014() bool { return m.serverVersion < 80014 } +func (m *MySQLLexerBase) isServerVersionGe80014() bool { return m.serverVersion >= 80014 } +func (m *MySQLLexerBase) isServerVersionGe80017() bool { return m.serverVersion >= 80017 } +func (m *MySQLLexerBase) isServerVersionGe80018() bool { return m.serverVersion >= 80018 } +func (m *MySQLLexerBase) isMasterCompressionAlgorithm() bool { return m.serverVersion >= 80018 && m.isServerVersionLt80024() } +func (m *MySQLLexerBase) isServerVersionLt80031() bool { return m.serverVersion < 80031 } + +func (m *MySQLLexerBase) doLogicalOr() { + if m.isSqlModeActive(PipesAsConcat) { + m.SetType(MySQLLexerCONCAT_PIPES_SYMBOL) + } else { + m.SetType(MySQLLexerLOGICAL_OR_OPERATOR) + } +} + +func (m *MySQLLexerBase) isSqlModeActive(mode SqlMode) bool { + return m.sqlModes[mode] +} + + +func (m *MySQLLexerBase) doIntNumber() { + m.SetType(m.determineNumericType(m.GetText())); +} + +func (m *MySQLLexerBase) determineNumericType(text string) int { + length := len(text) - 1 + if length < StaticMySQLLexerBase.longLength { + return MySQLLexerINT_NUMBER + } + + negative := false + index := 0 + if text[index] == '+' { + index++ + length-- + } else if text[index] == '-' { + index++ + length-- + negative = true + } + for text[index] == '0' && length > 0 { + index++ + length-- + } + + if length < StaticMySQLLexerBase.longLength { + return MySQLLexerINT_NUMBER + } + + var smaller int + var bigger int + var cmp string + if negative { + if length == StaticMySQLLexerBase.longLength { + cmp = StaticMySQLLexerBase.signedLongString[1:] + smaller = MySQLLexerINT_NUMBER + bigger = MySQLLexerLONG_NUMBER + } else if length < StaticMySQLLexerBase.signedLongLongLength { + return MySQLLexerLONG_NUMBER; + } else if length > StaticMySQLLexerBase.signedLongLongLength { + return MySQLLexerDECIMAL_NUMBER; + } else { + cmp = StaticMySQLLexerBase.signedLongLongString[1:] + smaller = MySQLLexerLONG_NUMBER + bigger = MySQLLexerDECIMAL_NUMBER + } + } else { + if length == StaticMySQLLexerBase.longLength { + cmp = StaticMySQLLexerBase.longString + smaller = MySQLLexerINT_NUMBER + bigger = MySQLLexerLONG_NUMBER + } else if length < StaticMySQLLexerBase.longLongLength { + return MySQLLexerLONG_NUMBER; + } else if length > StaticMySQLLexerBase.longLongLength { + if length > StaticMySQLLexerBase.unsignedLongLongLength { + return MySQLLexerDECIMAL_NUMBER; + } + cmp = StaticMySQLLexerBase.unsignedLongLongString[1:] + smaller = MySQLLexerULONGLONG_NUMBER + bigger = MySQLLexerDECIMAL_NUMBER + } else { + cmp = StaticMySQLLexerBase.longLongString + smaller = MySQLLexerLONG_NUMBER + bigger = MySQLLexerULONGLONG_NUMBER + } + } + + otherIndex := 0 + for index < len(text) && cmp[otherIndex] == text[index] { + index++ + otherIndex++ + } + if index < len(text) { + index++ + otherIndex++ + } + if text[index - 1] <= cmp[otherIndex - 1] { + return smaller + } + return bigger +} + +func (m *MySQLLexerBase) checkMySQLVersion(text string) bool { + if len(text) < 8 { // Minimum is: /*!12345 + return false + } + + // Skip version comment introducer. + version, err := strconv.Atoi(text[3:]) + if err != nil { + return false + } + + if version <= StaticMySQLLexerBase.serverVersion { + StaticMySQLLexerBase.inVersionComment = true + return true + } + + return false +} + +func (m *MySQLLexerBase) determineFunction(proposed int) int { + // Skip any whitespace character if the sql mode says they should be ignored, + // before actually trying to match the open parenthesis. + input := m.GetInputStream().LA(1) + if m.isSqlModeActive(IgnoreSpace) { + for input == ' ' || input == '\t' || input == '\r' || input == '\n' { + m.Interpreter.Consume(m.GetInputStream()) + // Update channel and token type + m.SetChannel(antlr.LexerHidden) + m.SetType(MySQLLexerWHITESPACE) + input = m.GetInputStream().LA(1) + } + } + + // Determine if the next character is an open parenthesis + if input == '(' { + return proposed + } + return MySQLLexerIDENTIFIER +} + +func (m *MySQLLexerBase) doAdddate() { m.SetType(m.determineFunction(MySQLLexerADDDATE_SYMBOL)) } +func (m *MySQLLexerBase) doBitAnd() { m.SetType(m.determineFunction(MySQLLexerBIT_AND_SYMBOL)) } +func (m *MySQLLexerBase) doBitOr() { m.SetType(m.determineFunction(MySQLLexerBIT_OR_SYMBOL)) } +func (m *MySQLLexerBase) doBitXor() { m.SetType(m.determineFunction(MySQLLexerBIT_XOR_SYMBOL)) } +func (m *MySQLLexerBase) doCast() { m.SetType(m.determineFunction(MySQLLexerCAST_SYMBOL)) } +func (m *MySQLLexerBase) doCount() { m.SetType(m.determineFunction(MySQLLexerCOUNT_SYMBOL)) } +func (m *MySQLLexerBase) doCurdate() { m.SetType(m.determineFunction(MySQLLexerCURDATE_SYMBOL)) } +func (m *MySQLLexerBase) doCurrentDate() { m.SetType(m.determineFunction(MySQLLexerCURDATE_SYMBOL)) } +func (m *MySQLLexerBase) doCurrentTime() { m.SetType(m.determineFunction(MySQLLexerCURTIME_SYMBOL)) } +func (m *MySQLLexerBase) doCurtime() { m.SetType(m.determineFunction(MySQLLexerCURTIME_SYMBOL)) } +func (m *MySQLLexerBase) doDateAdd() { m.SetType(m.determineFunction(MySQLLexerDATE_ADD_SYMBOL)) } +func (m *MySQLLexerBase) doDateSub() { m.SetType(m.determineFunction(MySQLLexerDATE_SUB_SYMBOL)) } +func (m *MySQLLexerBase) doExtract() { m.SetType(m.determineFunction(MySQLLexerEXTRACT_SYMBOL)) } + +func (m *MySQLLexerBase) doGroupConcat() { m.SetType(m.determineFunction(MySQLLexerGROUP_CONCAT_SYMBOL)) } +func (m *MySQLLexerBase) doMax() { m.SetType(m.determineFunction(MySQLLexerMAX_SYMBOL)) } +func (m *MySQLLexerBase) doMid() { m.SetType(m.determineFunction(MySQLLexerSUBSTRING_SYMBOL)) } +func (m *MySQLLexerBase) doMin() { m.SetType(m.determineFunction(MySQLLexerMIN_SYMBOL)) } + +func (m *MySQLLexerBase) doNot() { + if m.isSqlModeActive(HighNotPrecedence) { + m.SetType(MySQLLexerNOT2_SYMBOL) + } else { + m.SetType(MySQLLexerNOT_SYMBOL) + } +} + +func (m *MySQLLexerBase) doNow() { m.SetType(m.determineFunction(MySQLLexerNOW_SYMBOL)) } +func (m *MySQLLexerBase) doPosition() { m.SetType(m.determineFunction(MySQLLexerPOSITION_SYMBOL)) } +func (m *MySQLLexerBase) doSessionUser() { m.SetType(m.determineFunction(MySQLLexerUSER_SYMBOL)) } +func (m *MySQLLexerBase) doStddevSamp() { m.SetType(m.determineFunction(MySQLLexerSTDDEV_SAMP_SYMBOL)) } +func (m *MySQLLexerBase) doStddev() { m.SetType(m.determineFunction(MySQLLexerSTD_SYMBOL)) } +func (m *MySQLLexerBase) doStddevPop() { m.SetType(m.determineFunction(MySQLLexerSTD_SYMBOL)) } +func (m *MySQLLexerBase) doStd() { m.SetType(m.determineFunction(MySQLLexerSTD_SYMBOL)) } +func (m *MySQLLexerBase) doSubdate() { m.SetType(m.determineFunction(MySQLLexerSUBDATE_SYMBOL)) } +func (m *MySQLLexerBase) doSubstr() { m.SetType(m.determineFunction(MySQLLexerSUBSTRING_SYMBOL)) } +func (m *MySQLLexerBase) doSubstring() { m.SetType(m.determineFunction(MySQLLexerSUBSTRING_SYMBOL)) } +func (m *MySQLLexerBase) doSum() { m.SetType(m.determineFunction(MySQLLexerSUM_SYMBOL)) } +func (m *MySQLLexerBase) doSysdate() { m.SetType(m.determineFunction(MySQLLexerSYSDATE_SYMBOL)) } +func (m *MySQLLexerBase) doSystemUser() { m.SetType(m.determineFunction(MySQLLexerUSER_SYMBOL)) } +func (m *MySQLLexerBase) doTrim() { m.SetType(m.determineFunction(MySQLLexerTRIM_SYMBOL)) } +func (m *MySQLLexerBase) doVariance() { m.SetType(m.determineFunction(MySQLLexerVARIANCE_SYMBOL)) } +func (m *MySQLLexerBase) doVarPop() { m.SetType(m.determineFunction(MySQLLexerVARIANCE_SYMBOL)) } +func (m *MySQLLexerBase) doVarSamp() { m.SetType(m.determineFunction(MySQLLexerVAR_SAMP_SYMBOL)) } +func (m *MySQLLexerBase) doUnderscoreCharset() { m.SetType(m.checkCharset(m.GetText())) } +func (m *MySQLLexerBase) isVersionComment() bool { return m.checkMySQLVersion(m.GetText()) } +func (m *MySQLLexerBase) isBackTickQuotedId() bool { return ! m.isSqlModeActive(NoBackslashEscapes) } +func (m *MySQLLexerBase) isDoubleQuotedText() bool { return !m.isSqlModeActive(NoBackslashEscapes) } +func (m *MySQLLexerBase) isSingleQuotedText() bool { return !m.isSqlModeActive(NoBackslashEscapes) } +func (m *MySQLLexerBase) startInVersionComment() { m.inVersionComment = true } +func (m *MySQLLexerBase) endInVersionComment() { m.inVersionComment = false } +func (m *MySQLLexerBase) isInVersionComment() bool { return m.inVersionComment } + +/** + * Checks if the given text corresponds to a charset defined in the server (text is preceded by an underscore). + * + * @param text The text to check. + * + * @returns UNDERSCORE_CHARSET if so, otherwise IDENTIFIER. + */ +func (m *MySQLLexerBase) checkCharset(text string) int { + if _, ok := m.charSets[text]; ok { + return MySQLLexerUNDERSCORE_CHARSET + } else { + return MySQLLexerIDENTIFIER + } } diff --git a/sql/mysql/Oracle/Go/MySQLParserBase.go b/sql/mysql/Oracle/Go/MySQLParserBase.go index d7d6fc7092..83c0ba75b9 100644 --- a/sql/mysql/Oracle/Go/MySQLParserBase.go +++ b/sql/mysql/Oracle/Go/MySQLParserBase.go @@ -5,5 +5,43 @@ import ( ) type MySQLParserBase struct { - *antlr.BaseParser + *antlr.BaseParser + serverVersion int + sqlModes map[SqlMode]bool + supportMle bool } + +func NewMySQLParserBase(input antlr.InputStream) *MySQLParserBase { + r := &MySQLParserBase{ + supportMle: true, + serverVersion: 80200, + } + r.sqlModes = make(map[SqlMode]bool) + return r +} + +// isSqlModeActive determines if the given SQL mode is currently active in the lexer. +func (m *MySQLParserBase) isSqlModeActive(mode SqlMode) bool { + return m.sqlModes[mode] +} + +// isPureIdentifier checks if the lexer is in ANSI_QUOTES mode. +func (m *MySQLParserBase) isPureIdentifier() bool { + return m.isSqlModeActive(AnsiQuotes) +} + +// isTextStringLiteral checks if the lexer is not in ANSI_QUOTES mode. +func (m *MySQLParserBase) isTextStringLiteral() bool { + return !m.isSqlModeActive(AnsiQuotes) +} + +// isStoredRoutineBody checks if the server version supports stored routine body. +func (m *MySQLParserBase) isStoredRoutineBody() bool { + return m.serverVersion >= 80032 && m.supportMle +} + +// isSelectStatementWithInto checks if the server version supports SELECT INTO syntax. +func (m *MySQLParserBase) isSelectStatementWithInto() bool { + return m.serverVersion >= 80024 && m.serverVersion < 80031 +} + diff --git a/sql/mysql/Oracle/Go/Set.go b/sql/mysql/Oracle/Go/Set.go deleted file mode 100644 index 8945f56630..0000000000 --- a/sql/mysql/Oracle/Go/Set.go +++ /dev/null @@ -1,46 +0,0 @@ -package parser - -// Set is a type that represents a set implemented as a map -type Set[T comparable] struct { - items map[T]struct{} -} - -// NewSet creates and returns a new set -func NewSet[T comparable]() *Set[T] { - return &Set[T]{items: make(map[T]struct{})} -} - -// Add adds an element to the set -func (s *Set[T]) Add(element T) { - s.items[element] = struct{}{} -} - -// Remove removes an element from the set -func (s *Set[T]) Remove(element T) { - delete(s.items, element) -} - -// Contains checks if an element is in the set -func (s *Set[T]) Contains(element T) bool { - _, exists := s.items[element] - return exists -} - -// Size returns the number of elements in the set -func (s *Set[T]) Size() int { - return len(s.items) -} - -// Clear removes all elements from the set -func (s *Set[T]) Clear() { - s.items = make(map[T]struct{}) -} - -// Elements returns all elements in the set as a slice -func (s *Set[T]) Elements() []T { - elements := make([]T, 0, len(s.items)) - for key := range s.items { - elements = append(elements, key) - } - return elements -} diff --git a/sql/mysql/Oracle/MySQLLexer.g4 b/sql/mysql/Oracle/MySQLLexer.g4 index d481d9ad56..4adaa7a5b6 100644 --- a/sql/mysql/Oracle/MySQLLexer.g4 +++ b/sql/mysql/Oracle/MySQLLexer.g4 @@ -3657,7 +3657,8 @@ MYSQL_COMMENT_START ; VERSION_COMMENT_END - : '*/' {this.isInVersionComment()}? { this.endInVersionComment(); } -> channel(HIDDEN) + : '*/' {this.isInVersionComment()}? + { this.endInVersionComment(); } -> channel(HIDDEN) ; BLOCK_COMMENT From 725087106f8f8a247476b063bd51b084ea911a7a Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 11:38:19 -0500 Subject: [PATCH 14/29] Updates for Go port (incomplete still). --- sql/mysql/Oracle/Go/MySQLLexerBase.go | 34 +++++++++++++++++++++++--- sql/mysql/Oracle/Go/MySQLParserBase.go | 12 ++++++++- sql/mysql/Oracle/Go/SqlModes.go | 32 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 sql/mysql/Oracle/Go/SqlModes.go diff --git a/sql/mysql/Oracle/Go/MySQLLexerBase.go b/sql/mysql/Oracle/Go/MySQLLexerBase.go index 4f7057cb62..a98e5b8f33 100644 --- a/sql/mysql/Oracle/Go/MySQLLexerBase.go +++ b/sql/mysql/Oracle/Go/MySQLLexerBase.go @@ -5,6 +5,7 @@ import ( "github.com/antlr4-go/antlr/v4" ) + type MySQLLexerBase struct { *antlr.BaseLexer serverVersion int @@ -43,6 +44,7 @@ func init() { unsignedLongLongString: "18446744073709551615", unsignedLongLongLength: 20, } + StaticMySQLLexerBase.sqlModes = sqlModeFromString("ANSI_QUOTES"); } func NewMySQLLexerBase(input antlr.CharStream) *MySQLLexerBase { @@ -61,7 +63,7 @@ func NewMySQLLexerBase(input antlr.CharStream) *MySQLLexerBase { unsignedLongLongString: "18446744073709551615", unsignedLongLongLength: 20, } - r.sqlModes = make(map[SqlMode]bool) + r.sqlModes = sqlModeFromString("ANSI_QUOTES"); return r } @@ -86,8 +88,8 @@ func (l *MySQLLexerBase) MakeCommonToken(ttype int, text string) antlr.Token { func (m *MySQLLexerBase) emitDot() { m.pendingTokens = append(m.pendingTokens, m.MakeCommonToken(MySQLLexerDOT_SYMBOL, m.GetText())) -// ++this.Column; -// m.justEmitedDot = true + m.TokenStartColumn = m.TokenStartColumn + 1 + m.TokenStartCharIndex = m.TokenStartCharIndex + 1 } func (m *MySQLLexerBase) isServerVersionLt80024() bool { return m.serverVersion < 80024 } @@ -110,7 +112,7 @@ func (m *MySQLLexerBase) doLogicalOr() { } func (m *MySQLLexerBase) isSqlModeActive(mode SqlMode) bool { - return m.sqlModes[mode] + return StaticMySQLLexerBase.sqlModes[mode] } @@ -304,3 +306,27 @@ func (m *MySQLLexerBase) checkCharset(text string) int { } } +/** + * Implements the multi token feature required in our lexer. + * A lexer rule can emit more than a single token, if needed. + * + * @returns The next token in the token stream. + */ +func (m *MySQLLexerBase) NextToken() antlr.Token { + if len(m.pendingTokens) != 0 { + pending := m.pendingTokens[0] + m.pendingTokens = m.pendingTokens[1:] + return pending + } + + // Let the main lexer class run the next token recognition. + // This might create additional tokens again. + next := m.BaseLexer.NextToken() // Get next token + if len(m.pendingTokens) != 0 { + pending := m.pendingTokens[0] + m.pendingTokens = m.pendingTokens[1:] + m.pendingTokens = append(m.pendingTokens, next) + return pending + } + return next +} diff --git a/sql/mysql/Oracle/Go/MySQLParserBase.go b/sql/mysql/Oracle/Go/MySQLParserBase.go index 83c0ba75b9..268549f0fb 100644 --- a/sql/mysql/Oracle/Go/MySQLParserBase.go +++ b/sql/mysql/Oracle/Go/MySQLParserBase.go @@ -11,6 +11,16 @@ type MySQLParserBase struct { supportMle bool } +var StaticMySQLParserBase MySQLParserBase + +func init() { + StaticMySQLParserBase = MySQLParserBase { + supportMle: true, + serverVersion: 80200, + } + StaticMySQLParserBase.sqlModes = sqlModeFromString("ANSI_QUOTES"); +} + func NewMySQLParserBase(input antlr.InputStream) *MySQLParserBase { r := &MySQLParserBase{ supportMle: true, @@ -22,7 +32,7 @@ func NewMySQLParserBase(input antlr.InputStream) *MySQLParserBase { // isSqlModeActive determines if the given SQL mode is currently active in the lexer. func (m *MySQLParserBase) isSqlModeActive(mode SqlMode) bool { - return m.sqlModes[mode] + return StaticMySQLParserBase.sqlModes[mode] } // isPureIdentifier checks if the lexer is in ANSI_QUOTES mode. diff --git a/sql/mysql/Oracle/Go/SqlModes.go b/sql/mysql/Oracle/Go/SqlModes.go new file mode 100644 index 0000000000..caf1184b57 --- /dev/null +++ b/sql/mysql/Oracle/Go/SqlModes.go @@ -0,0 +1,32 @@ +package parser + +import ( + "strings" +) + +func sqlModeFromString(modes string) map[SqlMode]bool { + result := make(map[SqlMode]bool) + parts := strings.Split(strings.ToUpper(modes), ",") + + for _, mode := range parts { + switch mode { + case "ANSI", "DB2", "MAXDB", "MSSQL", "ORACLE", "POSTGRESQL": + result[AnsiQuotes] = true + result[PipesAsConcat] = true + result[IgnoreSpace] = true + case "ANSI_QUOTES": + result[AnsiQuotes] = true + case "PIPES_AS_CONCAT": + result[PipesAsConcat] = true + case "NO_BACKSLASH_ESCAPES": + result[NoBackslashEscapes] = true + case "IGNORE_SPACE": + result[IgnoreSpace] = true + case "HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40": + result[HighNotPrecedence] = true + } + } + + return result +} + From e988c939ae81fe4263fd11dd03b46ffcb2ca41af Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 13:43:59 -0500 Subject: [PATCH 15/29] Fix properties of tokens with doEmit() and following token. It should probably not be done this way, but this does work, and the properties for indices, line and column, and text are now all correct. *Original source code by Mike has bugs.* --- sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts index 7de332f036..c0b5bc7be2 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLLexerBase.ts @@ -22,6 +22,7 @@ export abstract class MySQLLexerBase extends Lexer { protected inVersionComment = false; private pendingTokens: Token[] = []; + private justEmittedDot: boolean; static #longString = "2147483647"; static #longLength = 10; @@ -77,7 +78,6 @@ export abstract class MySQLLexerBase extends Lexer { pending = this.pendingTokens.shift(); if (pending) { this.pendingTokens.push(next); - return pending; } @@ -99,7 +99,6 @@ export abstract class MySQLLexerBase extends Lexer { const version = parseInt(text.substring(3), 10); if (version <= this.serverVersion) { this.inVersionComment = true; - return true; } @@ -229,12 +228,21 @@ export abstract class MySQLLexerBase extends Lexer { */ protected emitDot(): void { this.pendingTokens.push(this.tokenFactory.create([this, this.inputStream], MySQLLexer.DOT_SYMBOL, - this.text, this.channel, this.tokenStartCharIndex, this.tokenStartCharIndex, this.line, - this.column, + ".", this.channel, this.tokenStartCharIndex, this.tokenStartCharIndex, this.line, + this.column - this.text.length, )); - - ++this.column; ++this.tokenStartCharIndex; + this.justEmittedDot = true; + } + + public override emit(): Token + { + let t = super.emit(); + if (this.justEmittedDot) { + t.column = t.column + 1; + this.justEmittedDot = false; + } + return t; } public isServerVersionLt80024(): boolean From 7ff2a3a539fb95aa3f1d466c4f7e8b275f68faf9 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 16:14:44 -0500 Subject: [PATCH 16/29] Fix Cpp token BS, now identical to Antlr4ng and correct. --- sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp | 140 +++++++++++++----------- sql/mysql/Oracle/Cpp/MySQLLexerBase.h | 7 +- 2 files changed, 81 insertions(+), 66 deletions(-) diff --git a/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp b/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp index d8656b56c3..2f6ca5606b 100644 --- a/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp +++ b/sql/mysql/Oracle/Cpp/MySQLLexerBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include MySQLLexerBase::MySQLLexerBase(antlr4::CharStream * input) : antlr4::Lexer(input) { @@ -30,17 +31,17 @@ void MySQLLexerBase::reset() { std::unique_ptr MySQLLexerBase::nextToken() { if (!this->pendingTokens.empty()) { - std::unique_ptr pending = std::move(this->pendingTokens.front()); - this->pendingTokens.pop(); - return std::move(pending); + std::unique_ptr pending = std::move(this->pendingTokens.front()); + this->pendingTokens.pop(); + return std::move(pending); } - std::unique_ptr next = std::move(Lexer::nextToken()); + std::unique_ptr next = std::move(Lexer::nextToken()); if (!pendingTokens.empty()) { - std::unique_ptr pending = std::move(this->pendingTokens.front()); - this->pendingTokens.pop(); + std::unique_ptr pending = std::move(this->pendingTokens.front()); + this->pendingTokens.pop(); this->pendingTokens.push(std::move(next)); - return std::move(pending); + return std::move(pending); } return std::move(next); @@ -139,49 +140,62 @@ int MySQLLexerBase::checkCharset(std::string text) { } void MySQLLexerBase::emitDot() { - auto * t = new antlr4::CommonToken( + auto len = this->getText().length(); + auto * t = new antlr4::CommonToken( std::pair(this, this->_input), MySQLLexer::DOT_SYMBOL, 0, this->tokenStartCharIndex, this->tokenStartCharIndex); - pendingTokens.push(std::make_unique(t)); - this->setCharPositionInLine(this->getCharPositionInLine() + 1); - + t->setCharPositionInLine(t->getCharPositionInLine() - len); + pendingTokens.push(std::make_unique(t)); + this->setCharPositionInLine(this->getCharPositionInLine() + 1); ++this->tokenStartCharIndex; + this->justEmittedDot = true; +} + +antlr4::Token* MySQLLexerBase::emit() { + auto t = Lexer::emit(); + if (this->justEmittedDot) { + antlr4::CommonToken* p = dynamic_cast(t); + p->setCharPositionInLine(p->getCharPositionInLine() + 1); + this->setCharPositionInLine(this->getCharPositionInLine() - 1); + this->justEmittedDot = false; + } + return t; } bool MySQLLexerBase::isServerVersionLt80024() { - return this->serverVersion < 80024; + return this->serverVersion < 80024; } bool MySQLLexerBase::isServerVersionGe80024() { - return this->serverVersion >= 80024; + return this->serverVersion >= 80024; } bool MySQLLexerBase::isServerVersionGe80011() { - return this->serverVersion >= 80011; + return this->serverVersion >= 80011; } bool MySQLLexerBase::isServerVersionGe80013() { - return this->serverVersion >= 80013; + return this->serverVersion >= 80013; } bool MySQLLexerBase::isServerVersionLt80014() { - return this->serverVersion < 80014; + return this->serverVersion < 80014; } bool MySQLLexerBase::isServerVersionGe80014() { - return this->serverVersion >= 80014; + return this->serverVersion >= 80014; } bool MySQLLexerBase::isServerVersionGe80017() { - return this->serverVersion >= 80017; + return this->serverVersion >= 80017; } bool MySQLLexerBase::isServerVersionGe80018() { return this->serverVersion >= 80018; } @@ -189,196 +203,196 @@ bool MySQLLexerBase::isServerVersionGe80018() { return this->serverVersion >= 80 bool MySQLLexerBase::isMasterCompressionAlgorithm() { return this->serverVersion >= 80018 && this->isServerVersionLt80024(); } bool MySQLLexerBase::isServerVersionLt80031() { - return this->serverVersion < 80031; + return this->serverVersion < 80031; } void MySQLLexerBase::doLogicalOr() { - this->type = this->isSqlModeActive(SqlMode::PipesAsConcat) ? MySQLLexer::CONCAT_PIPES_SYMBOL : MySQLLexer::LOGICAL_OR_OPERATOR; + this->type = this->isSqlModeActive(SqlMode::PipesAsConcat) ? MySQLLexer::CONCAT_PIPES_SYMBOL : MySQLLexer::LOGICAL_OR_OPERATOR; } void MySQLLexerBase::doIntNumber() { - this->type = this->determineNumericType(this->getText()); + this->type = this->determineNumericType(this->getText()); } void MySQLLexerBase::doAdddate() { - this->type = this->determineFunction(MySQLLexer::ADDDATE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::ADDDATE_SYMBOL); } void MySQLLexerBase::doBitAnd() { - this->type = this->determineFunction(MySQLLexer::BIT_AND_SYMBOL); + this->type = this->determineFunction(MySQLLexer::BIT_AND_SYMBOL); } void MySQLLexerBase::doBitOr() { - this->type = this->determineFunction(MySQLLexer::BIT_OR_SYMBOL); + this->type = this->determineFunction(MySQLLexer::BIT_OR_SYMBOL); } void MySQLLexerBase::doBitXor() { - this->type = this->determineFunction(MySQLLexer::BIT_XOR_SYMBOL); + this->type = this->determineFunction(MySQLLexer::BIT_XOR_SYMBOL); } void MySQLLexerBase::doCast() { - this->type = this->determineFunction(MySQLLexer::CAST_SYMBOL); + this->type = this->determineFunction(MySQLLexer::CAST_SYMBOL); } void MySQLLexerBase::doCount() { - this->type = this->determineFunction(MySQLLexer::COUNT_SYMBOL); + this->type = this->determineFunction(MySQLLexer::COUNT_SYMBOL); } void MySQLLexerBase::doCurdate() { - this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); } void MySQLLexerBase::doCurrentDate() { - this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::CURDATE_SYMBOL); } void MySQLLexerBase::doCurrentTime() { - this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); + this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); } void MySQLLexerBase::doCurtime() { - this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); + this->type = this->determineFunction(MySQLLexer::CURTIME_SYMBOL); } void MySQLLexerBase::doDateAdd() { - this->type = this->determineFunction(MySQLLexer::DATE_ADD_SYMBOL); + this->type = this->determineFunction(MySQLLexer::DATE_ADD_SYMBOL); } void MySQLLexerBase::doDateSub() { - this->type = this->determineFunction(MySQLLexer::DATE_SUB_SYMBOL); + this->type = this->determineFunction(MySQLLexer::DATE_SUB_SYMBOL); } void MySQLLexerBase::doExtract() { - this->type = this->determineFunction(MySQLLexer::EXTRACT_SYMBOL); + this->type = this->determineFunction(MySQLLexer::EXTRACT_SYMBOL); } void MySQLLexerBase::doGroupConcat() { - this->type = this->determineFunction(MySQLLexer::GROUP_CONCAT_SYMBOL); + this->type = this->determineFunction(MySQLLexer::GROUP_CONCAT_SYMBOL); } void MySQLLexerBase::doMax() { - this->type = this->determineFunction(MySQLLexer::MAX_SYMBOL); + this->type = this->determineFunction(MySQLLexer::MAX_SYMBOL); } void MySQLLexerBase::doMid() { - this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); } void MySQLLexerBase::doMin() { - this->type = this->determineFunction(MySQLLexer::MIN_SYMBOL); + this->type = this->determineFunction(MySQLLexer::MIN_SYMBOL); } void MySQLLexerBase::doNot() { - this->type = this->isSqlModeActive(SqlMode::HighNotPrecedence) ? MySQLLexer::NOT2_SYMBOL: MySQLLexer::NOT_SYMBOL; + this->type = this->isSqlModeActive(SqlMode::HighNotPrecedence) ? MySQLLexer::NOT2_SYMBOL: MySQLLexer::NOT_SYMBOL; } void MySQLLexerBase::doNow() { - this->type = this->determineFunction(MySQLLexer::NOW_SYMBOL); + this->type = this->determineFunction(MySQLLexer::NOW_SYMBOL); } void MySQLLexerBase::doPosition() { - this->type = this->determineFunction(MySQLLexer::POSITION_SYMBOL); + this->type = this->determineFunction(MySQLLexer::POSITION_SYMBOL); } void MySQLLexerBase::doSessionUser() { - this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); + this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); } void MySQLLexerBase::doStddevSamp() { - this->type = this->determineFunction(MySQLLexer::STDDEV_SAMP_SYMBOL); + this->type = this->determineFunction(MySQLLexer::STDDEV_SAMP_SYMBOL); } void MySQLLexerBase::doStddev() { - this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); } void MySQLLexerBase::doStddevPop() { - this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); } void MySQLLexerBase::doStd() { - this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); + this->type = this->determineFunction(MySQLLexer::STD_SYMBOL); } void MySQLLexerBase::doSubdate() { - this->type = this->determineFunction(MySQLLexer::SUBDATE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SUBDATE_SYMBOL); } void MySQLLexerBase::doSubstr() { - this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); } void MySQLLexerBase::doSubstring() { - this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SUBSTRING_SYMBOL); } void MySQLLexerBase::doSum() { - this->type = this->determineFunction(MySQLLexer::SUM_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SUM_SYMBOL); } void MySQLLexerBase::doSysdate() { - this->type = this->determineFunction(MySQLLexer::SYSDATE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::SYSDATE_SYMBOL); } void MySQLLexerBase::doSystemUser() { - this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); + this->type = this->determineFunction(MySQLLexer::USER_SYMBOL); } void MySQLLexerBase::doTrim() { - this->type = this->determineFunction(MySQLLexer::TRIM_SYMBOL); + this->type = this->determineFunction(MySQLLexer::TRIM_SYMBOL); } void MySQLLexerBase::doVariance() { - this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); } void MySQLLexerBase::doVarPop() { - this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); + this->type = this->determineFunction(MySQLLexer::VARIANCE_SYMBOL); } void MySQLLexerBase::doVarSamp() { - this->type = this->determineFunction(MySQLLexer::VAR_SAMP_SYMBOL); + this->type = this->determineFunction(MySQLLexer::VAR_SAMP_SYMBOL); } void MySQLLexerBase::doUnderscoreCharset() { - this->type = this->checkCharset(this->getText()); + this->type = this->checkCharset(this->getText()); } bool MySQLLexerBase::isVersionComment() { @@ -387,32 +401,32 @@ bool MySQLLexerBase::isVersionComment() { bool MySQLLexerBase::isBackTickQuotedId() { - return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); } bool MySQLLexerBase::isDoubleQuotedText() { - return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); } bool MySQLLexerBase::isSingleQuotedText() { - return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); + return !this->isSqlModeActive(SqlMode::NoBackslashEscapes); } void MySQLLexerBase::startInVersionComment() { - inVersionComment = true; + inVersionComment = true; } void MySQLLexerBase::endInVersionComment() { - inVersionComment = false; + inVersionComment = false; } bool MySQLLexerBase::isInVersionComment() { - return inVersionComment; + return inVersionComment; } std::string MySQLLexerBase::longString = "2147483647"; diff --git a/sql/mysql/Oracle/Cpp/MySQLLexerBase.h b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h index 3f075e6ff6..098fac2fe4 100644 --- a/sql/mysql/Oracle/Cpp/MySQLLexerBase.h +++ b/sql/mysql/Oracle/Cpp/MySQLLexerBase.h @@ -30,7 +30,7 @@ class MySQLLexerBase : public antlr4::Lexer bool inVersionComment; private: - std::queue> pendingTokens; + std::queue> pendingTokens; static std::string longString; static int longLength; static std::string signedLongString; @@ -41,7 +41,7 @@ class MySQLLexerBase : public antlr4::Lexer static std::string unsignedLongLongString; static int unsignedLongLongLength; - bool justEmitedDot; + bool justEmittedDot; /** * Determines if the given SQL mode is currently active in the lexer. @@ -53,7 +53,8 @@ class MySQLLexerBase : public antlr4::Lexer public: bool isSqlModeActive(SqlMode mode); void reset(); - std::unique_ptr nextToken() override; + std::unique_ptr nextToken() override; + antlr4::Token * emit() override; protected: bool checkMySQLVersion(std::string text); From 5dcbf13ae73883c9b875bc98a4521eb2c49c7259 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 17:41:26 -0500 Subject: [PATCH 17/29] Fix explicit expressions in parser grammar, make target agnostic. --- sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts | 24 ++ sql/mysql/Oracle/MySQLParser.g4 | 234 +++++++++---------- 2 files changed, 141 insertions(+), 117 deletions(-) diff --git a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts index 89e637f844..9bf4b582de 100644 --- a/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts +++ b/sql/mysql/Oracle/Antlr4ng/MySQLParserBase.ts @@ -51,4 +51,28 @@ export abstract class MySQLParserBase extends Parser { { return this.serverVersion >= 80024 && this.serverVersion < 80031; } + + public isServerVersionGe80004(): boolean { return this.serverVersion >= 80004; } + public isServerVersionGe80011(): boolean { return this.serverVersion >= 80011; } + public isServerVersionGe80013(): boolean { return this.serverVersion >= 80013; } + public isServerVersionGe80014(): boolean { return this.serverVersion >= 80014; } + public isServerVersionGe80016(): boolean { return this.serverVersion >= 80016; } + public isServerVersionGe80017(): boolean { return this.serverVersion >= 80017; } + public isServerVersionGe80018(): boolean { return this.serverVersion >= 80018; } + public isServerVersionGe80019(): boolean { return this.serverVersion >= 80019; } + public isServerVersionGe80024(): boolean { return this.serverVersion >= 80024; } + public isServerVersionGe80025(): boolean { return this.serverVersion >= 80025; } + public isServerVersionGe80027(): boolean { return this.serverVersion >= 80027; } + public isServerVersionGe80031(): boolean { return this.serverVersion >= 80031; } + public isServerVersionGe80032(): boolean { return this.serverVersion >= 80032; } + public isServerVersionGe80100(): boolean { return this.serverVersion >= 80100; } + public isServerVersionGe80200(): boolean { return this.serverVersion >= 80200; } + public isServerVersionLt80011(): boolean { return this.serverVersion < 80011; } + public isServerVersionLt80012(): boolean { return this.serverVersion < 80012; } + public isServerVersionLt80014(): boolean { return this.serverVersion < 80014; } + public isServerVersionLt80016(): boolean { return this.serverVersion < 80016; } + public isServerVersionLt80017(): boolean { return this.serverVersion < 80017; } + public isServerVersionLt80024(): boolean { return this.serverVersion < 80024; } + public isServerVersionLt80025(): boolean { return this.serverVersion < 80025; } + public isServerVersionLt80031(): boolean { return this.serverVersion < 80031; } } diff --git a/sql/mysql/Oracle/MySQLParser.g4 b/sql/mysql/Oracle/MySQLParser.g4 index 4c7f4afcdf..742a7d64de 100644 --- a/sql/mysql/Oracle/MySQLParser.g4 +++ b/sql/mysql/Oracle/MySQLParser.g4 @@ -143,7 +143,7 @@ alterStatement | alterView | alterEvent | alterTablespace - | {this.serverVersion >= 80014}? alterUndoTablespace + | {this.isServerVersionGe80014()}? alterUndoTablespace | alterLogfileGroup | alterServer // ALTER USER is part of the user management rule. @@ -211,7 +211,7 @@ standaloneAlterCommands : DISCARD_SYMBOL TABLESPACE_SYMBOL | IMPORT_SYMBOL TABLESPACE_SYMBOL | alterPartition - | {this.serverVersion >= 80014}? ( + | {this.isServerVersionGe80014()}? ( SECONDARY_LOAD_SYMBOL | SECONDARY_UNLOAD_SYMBOL ) @@ -269,27 +269,27 @@ alterListItem | FOREIGN_SYMBOL KEY_SYMBOL columnInternalRef | PRIMARY_SYMBOL KEY_SYMBOL | keyOrIndex indexRef - | {this.serverVersion >= 80017}? CHECK_SYMBOL identifier - | {this.serverVersion >= 80019}? CONSTRAINT_SYMBOL identifier + | {this.isServerVersionGe80017()}? CHECK_SYMBOL identifier + | {this.isServerVersionGe80019()}? CONSTRAINT_SYMBOL identifier ) | DISABLE_SYMBOL KEYS_SYMBOL | ENABLE_SYMBOL KEYS_SYMBOL | ALTER_SYMBOL COLUMN_SYMBOL? columnInternalRef ( SET_SYMBOL DEFAULT_SYMBOL ( - {this.serverVersion >= 80014}? exprWithParentheses + {this.isServerVersionGe80014()}? exprWithParentheses | signedLiteralOrNull ) | DROP_SYMBOL DEFAULT_SYMBOL - | {this.serverVersion >= 80024}? SET_SYMBOL visibility + | {this.isServerVersionGe80024()}? SET_SYMBOL visibility ) | ALTER_SYMBOL INDEX_SYMBOL indexRef visibility - | {this.serverVersion >= 80017}? ALTER_SYMBOL CHECK_SYMBOL identifier constraintEnforcement - | {this.serverVersion >= 80019}? ALTER_SYMBOL CONSTRAINT_SYMBOL identifier constraintEnforcement + | {this.isServerVersionGe80017()}? ALTER_SYMBOL CHECK_SYMBOL identifier constraintEnforcement + | {this.isServerVersionGe80019()}? ALTER_SYMBOL CONSTRAINT_SYMBOL identifier constraintEnforcement | RENAME_SYMBOL COLUMN_SYMBOL columnInternalRef TO_SYMBOL identifier | RENAME_SYMBOL (TO_SYMBOL | AS_SYMBOL)? tableName | RENAME_SYMBOL keyOrIndex indexRef TO_SYMBOL indexName | CONVERT_SYMBOL TO_SYMBOL charset ( - {this.serverVersion >= 80014}? DEFAULT_SYMBOL + {this.isServerVersionGe80014()}? DEFAULT_SYMBOL | charsetName ) collate? | FORCE_SYMBOL @@ -340,7 +340,7 @@ alterTablespace : TABLESPACE_SYMBOL tablespaceRef ( (ADD_SYMBOL | DROP_SYMBOL) DATAFILE_SYMBOL textLiteral alterTablespaceOptions? | RENAME_SYMBOL TO_SYMBOL identifier - | {this.serverVersion >= 80014}? alterTablespaceOptions + | {this.isServerVersionGe80014()}? alterTablespaceOptions ) ; @@ -370,7 +370,7 @@ alterTablespaceOption | tsOptionEngine | tsOptionWait | tsOptionEncryption - | {this.serverVersion >= 80024}? tsOptionEngineAttribute + | {this.isServerVersionGe80024()}? tsOptionEngineAttribute ; changeTablespaceOption @@ -399,7 +399,7 @@ viewCheckOption alterInstanceStatement : INSTANCE_SYMBOL ROTATE_SYMBOL textOrIdentifier MASTER_SYMBOL KEY_SYMBOL - | {this.serverVersion >= 80024}? ( + | {this.isServerVersionGe80024()}? ( RELOAD_SYMBOL TLS_SYMBOL ( NO_SYMBOL ROLLBACK_SYMBOL ON_SYMBOL ERROR_SYMBOL | FOR_SYMBOL CHANNEL_SYMBOL identifier ( @@ -428,8 +428,8 @@ createStatement | createTablespace | createEvent | createRole - | {this.serverVersion >= 80011}? createSpatialReference - | {this.serverVersion >= 80014}? createUndoTablespace + | {this.isServerVersionGe80011()}? createSpatialReference + | {this.isServerVersionGe80014()}? createUndoTablespace ) ; @@ -440,7 +440,7 @@ createDatabase createDatabaseOption : defaultCharset | defaultCollation - | {this.serverVersion >= 80016}? defaultEncryption + | {this.isServerVersionGe80016()}? defaultEncryption ; createTable @@ -529,7 +529,7 @@ routineOption : option = COMMENT_SYMBOL textLiteral | option = LANGUAGE_SYMBOL ( SQL_SYMBOL - | {this.serverVersion >= 80032}? identifier + | {this.isServerVersionGe80032()}? identifier ) | option = NO_SYMBOL SQL_SYMBOL | option = CONTAINS_SYMBOL SQL_SYMBOL @@ -607,7 +607,7 @@ createUndoTablespace tsDataFileName : ADD_SYMBOL tsDataFile - | {this.serverVersion >= 80014}? (ADD_SYMBOL tsDataFile)? // now optional + | {this.isServerVersionGe80014()}? (ADD_SYMBOL tsDataFile)? // now optional ; tsDataFile @@ -628,7 +628,7 @@ tablespaceOption | tsOptionWait | tsOptionComment | tsOptionFileblockSize - | {this.serverVersion >= 80014}? tsOptionEncryption + | {this.isServerVersionGe80014()}? tsOptionEncryption ; tsOptionInitialSize @@ -754,8 +754,8 @@ dropStatement | dropTrigger | dropView | dropRole - | {this.serverVersion >= 80011}? dropSpatialReference - | {this.serverVersion >= 80014}? dropUndoTablespace + | {this.isServerVersionGe80011()}? dropSpatialReference + | {this.isServerVersionGe80014()}? dropUndoTablespace ) ; @@ -861,7 +861,7 @@ deleteStatement : withClause? DELETE_SYMBOL deleteStatementOption* ( FROM_SYMBOL ( tableAliasRefList USING_SYMBOL tableReferenceList whereClause? // Multi table variant 1. - | tableRef ({this.serverVersion >= 80017}? tableAlias)? partitionDelete? whereClause? orderClause? simpleLimitClause? + | tableRef ({this.isServerVersionGe80017()}? tableAlias)? partitionDelete? whereClause? orderClause? simpleLimitClause? // Single table delete. ) | tableAliasRefList FROM_SYMBOL tableReferenceList whereClause? // Multi table variant 2. @@ -954,7 +954,7 @@ values ; valuesReference - : { this.serverVersion >= 80018}? AS_SYMBOL identifier columnInternalRefList? + : {this.isServerVersionGe80018()}? AS_SYMBOL identifier columnInternalRefList? ; insertUpdateList @@ -982,23 +982,23 @@ loadDataLock ; loadFrom - : {this.serverVersion >= 80200}? FROM_SYMBOL + : {this.isServerVersionGe80200()}? FROM_SYMBOL ; loadSourceType : INFILE_SYMBOL - | {this.serverVersion >= 80200}? (URL_SYMBOL | S3_SYMBOL) + | {this.isServerVersionGe80200()}? (URL_SYMBOL | S3_SYMBOL) ; sourceCount - : {this.serverVersion >= 80200}? ( + : {this.isServerVersionGe80200()}? ( COUNT_SYMBOL INT_NUMBER | pureIdentifier INT_NUMBER ) ; sourceOrder - : {this.serverVersion >= 80200}? IN_SYMBOL PRIMARY_SYMBOL KEY_SYMBOL ORDER_SYMBOL + : {this.isServerVersionGe80200()}? IN_SYMBOL PRIMARY_SYMBOL KEY_SYMBOL ORDER_SYMBOL ; xmlRowsIdentifiedBy @@ -1027,15 +1027,15 @@ fieldOrVariableList ; loadAlgorithm - : {this.serverVersion >= 80200}? ALGORITHM_SYMBOL EQUAL_OPERATOR BULK_SYMBOL + : {this.isServerVersionGe80200()}? ALGORITHM_SYMBOL EQUAL_OPERATOR BULK_SYMBOL ; loadParallel - : {this.serverVersion >= 80200}? PARALLEL_SYMBOL EQUAL_OPERATOR INT_NUMBER + : {this.isServerVersionGe80200()}? PARALLEL_SYMBOL EQUAL_OPERATOR INT_NUMBER ; loadMemory - : {this.serverVersion >= 80200}? MEMORY_SYMBOL EQUAL_OPERATOR sizeNumber + : {this.isServerVersionGe80200()}? MEMORY_SYMBOL EQUAL_OPERATOR sizeNumber ; //---------------------------------------------------------------------------------------------------------------------- @@ -1073,7 +1073,7 @@ queryExpressionBody ( ( UNION_SYMBOL - | {this.serverVersion >= 80031}? (EXCEPT_SYMBOL | INTERSECT_SYMBOL) + | {this.isServerVersionGe80031()}? (EXCEPT_SYMBOL | INTERSECT_SYMBOL) ) unionOption? queryExpressionBody )* ; @@ -1084,8 +1084,8 @@ queryExpressionParens queryPrimary : querySpecification - | {this.serverVersion >= 80019}? tableValueConstructor - | {this.serverVersion >= 80019}? explicitTable + | {this.isServerVersionGe80019()}? tableValueConstructor + | {this.isServerVersionGe80019()}? explicitTable ; querySpecification @@ -1144,7 +1144,7 @@ havingClause ; qualifyClause - : {this.serverVersion >= 80200}? QUALIFY_SYMBOL expr + : {this.isServerVersionGe80200()}? QUALIFY_SYMBOL expr ; windowClause @@ -1219,7 +1219,7 @@ commonTableExpression groupByClause : GROUP_SYMBOL BY_SYMBOL orderList olapOption? - | {this.serverVersion >= 80032}? GROUP_SYMBOL BY_SYMBOL ( + | {this.isServerVersionGe80032()}? GROUP_SYMBOL BY_SYMBOL ( ROLLUP_SYMBOL | CUBE_SYMBOL ) OPEN_PAR_SYMBOL groupList CLOSE_PAR_SYMBOL @@ -1264,7 +1264,7 @@ selectOption ; lockingClauseList - : {this.serverVersion >= 80031}? lockingClause+ + : {this.isServerVersionGe80031()}? lockingClause+ ; lockingClause @@ -1304,7 +1304,7 @@ tableReference ( tableFactor // ODBC syntax - | OPEN_CURLY_SYMBOL ({this.serverVersion < 80017}? identifier | OJ_SYMBOL) escapedTableReference CLOSE_CURLY_SYMBOL + | OPEN_CURLY_SYMBOL ({this.isServerVersionGe80017()}? identifier | OJ_SYMBOL) escapedTableReference CLOSE_CURLY_SYMBOL ) joinedTable* ; @@ -1344,7 +1344,7 @@ tableFactor | singleTableParens | derivedTable | tableReferenceListParens - | {this.serverVersion >= 80004}? tableFunction + | {this.isServerVersionGe80004()}? tableFunction ; singleTable @@ -1357,7 +1357,7 @@ singleTableParens derivedTable : subquery tableAlias? columnInternalRefList? - | {this.serverVersion >= 80014}? LATERAL_SYMBOL subquery tableAlias? columnInternalRefList? + | {this.isServerVersionGe80014()}? LATERAL_SYMBOL subquery tableAlias? columnInternalRefList? ; // This rule covers both: joined_table_parens and table_reference_list_parens from sql_yacc.yy. @@ -1376,7 +1376,7 @@ columnsClause jtColumn : identifier FOR_SYMBOL ORDINALITY_SYMBOL - | identifier dataType ({this.serverVersion >= 80014}? collate)? EXISTS_SYMBOL? PATH_SYMBOL textStringLiteral + | identifier dataType ({this.isServerVersionGe80014()}? collate)? EXISTS_SYMBOL? PATH_SYMBOL textStringLiteral onEmptyOrErrorJsonTable? | NESTED_SYMBOL PATH_SYMBOL textStringLiteral columnsClause ; @@ -1412,7 +1412,7 @@ unionOption ; tableAlias - : (AS_SYMBOL | {this.serverVersion < 80017}? EQUAL_OPERATOR)? identifier + : (AS_SYMBOL | {this.isServerVersionGe80017()}? EQUAL_OPERATOR)? identifier ; indexHintList @@ -1557,7 +1557,7 @@ resetOption masterOrBinaryLogsAndGtids : MASTER_SYMBOL - | {this.serverVersion >= 80032}? BINARY_SYMBOL LOGS_SYMBOL AND_SYMBOL GTIDS_SYMBOL + | {this.isServerVersionGe80032()}? BINARY_SYMBOL LOGS_SYMBOL AND_SYMBOL GTIDS_SYMBOL ; sourceResetOptions @@ -1570,7 +1570,7 @@ replicationLoad changeReplicationSource : MASTER_SYMBOL - | {this.serverVersion >= 80024}? REPLICATION_SYMBOL SOURCE_SYMBOL + | {this.isServerVersionGe80024()}? REPLICATION_SYMBOL SOURCE_SYMBOL ; changeSource @@ -1613,10 +1613,10 @@ sourceDefinition | PRIVILEGE_CHECKS_USER_SYMBOL EQUAL_OPERATOR privilegeCheckDef | REQUIRE_ROW_FORMAT_SYMBOL EQUAL_OPERATOR ulong_number | REQUIRE_TABLE_PRIMARY_KEY_CHECK_SYMBOL EQUAL_OPERATOR tablePrimaryKeyCheckDef - | {this.serverVersion >= 80024}? SOURCE_CONNECTION_AUTO_FAILOVER_SYMBOL EQUAL_OPERATOR real_ulong_number - | {this.serverVersion >= 80024}? ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS_SYMBOL EQUAL_OPERATOR + | {this.isServerVersionGe80024()}? SOURCE_CONNECTION_AUTO_FAILOVER_SYMBOL EQUAL_OPERATOR real_ulong_number + | {this.isServerVersionGe80024()}? ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS_SYMBOL EQUAL_OPERATOR assignGtidsToAnonymousTransactionsDefinition - | {this.serverVersion >= 80027}? GTID_ONLY_SYMBOL EQUAL_OPERATOR real_ulong_number + | {this.isServerVersionGe80027()}? GTID_ONLY_SYMBOL EQUAL_OPERATOR real_ulong_number | sourceFileDef ; @@ -1922,7 +1922,7 @@ cloneStatement LOCAL_SYMBOL DATA_SYMBOL DIRECTORY_SYMBOL equal? textStringLiteral // Clone remote has been removed in 8.0.14. This alt is taken out by the conditional REMOTE_SYMBOL. | REMOTE_SYMBOL (FOR_SYMBOL REPLICATION_SYMBOL)? - | {this.serverVersion >= 80014}? INSTANCE_SYMBOL FROM_SYMBOL user COLON_SYMBOL ulong_number IDENTIFIED_SYMBOL BY_SYMBOL + | {this.isServerVersionGe80014()}? INSTANCE_SYMBOL FROM_SYMBOL user COLON_SYMBOL ulong_number IDENTIFIED_SYMBOL BY_SYMBOL textStringLiteral dataDirSSL? ) ; @@ -1952,8 +1952,8 @@ accountManagementStatement alterUserStatement : ALTER_SYMBOL USER_SYMBOL ifExists? ( ( - {this.serverVersion < 80014}? createUserList - | {this.serverVersion >= 80014}? alterUserList + {this.isServerVersionGe80014()}? createUserList + | {this.isServerVersionGe80014()}? alterUserList ) createUserTail | userFunction ( (identifiedByRandomPassword | identifiedByPassword) replacePassword? retainCurrentPassword? @@ -1972,8 +1972,8 @@ alterUserList ; alterUser - : {this.serverVersion < 80025}? oldAlterUser - | {this.serverVersion >= 80025}? ( + : {this.isServerVersionLt80025()}? oldAlterUser + | {this.isServerVersionGe80025()}? ( user ( identifiedByPassword ( REPLACE_SYMBOL textStringLiteral retainCurrentPassword? @@ -2027,7 +2027,7 @@ createUserStatement createUserTail : requireClause? connectOptions? accountLockPasswordExpireOptions* ( - {this.serverVersion >= 80024}? userAttributes + {this.isServerVersionGe80024()}? userAttributes )? ; @@ -2069,7 +2069,7 @@ accountLockPasswordExpireOptions real_ulong_number DAY_SYMBOL | DEFAULT_SYMBOL ) - | {this.serverVersion >= 80014}? REQUIRE_SYMBOL CURRENT_SYMBOL ( + | {this.isServerVersionGe80014()}? REQUIRE_SYMBOL CURRENT_SYMBOL ( DEFAULT_SYMBOL | OPTIONAL_SYMBOL )? @@ -2102,8 +2102,8 @@ grantStatement ; grantTargetList - : {this.serverVersion < 80011}? createUserList - | {this.serverVersion >= 80011}? userList + : {this.isServerVersionLt80011()}? createUserList + | {this.isServerVersionGe80011()}? userList ; grantOptions @@ -2128,7 +2128,7 @@ grantAs ; versionedRequireClause - : {this.serverVersion < 80011}? requireClause + : {this.isServerVersionLt80011()}? requireClause ; renameUserStatement @@ -2138,7 +2138,7 @@ renameUserStatement ; revokeStatement - : REVOKE_SYMBOL ({this.serverVersion >= 80031}? ifExists)? ( + : REVOKE_SYMBOL ({this.isServerVersionGe80031()}? ifExists)? ( roleOrPrivilegesList FROM_SYMBOL userList | roleOrPrivilegesList ON_SYMBOL aclType? grantIdentifier FROM_SYMBOL userList | ALL_SYMBOL PRIVILEGES_SYMBOL? ( @@ -2146,7 +2146,7 @@ revokeStatement | COMMA_SYMBOL GRANT_SYMBOL OPTION_SYMBOL ) FROM_SYMBOL userList | PROXY_SYMBOL ON_SYMBOL user FROM_SYMBOL userList - ) ({this.serverVersion >= 80031}? ignoreUnknownUser)? + ) ({this.isServerVersionGe80031()}? ignoreUnknownUser)? ; aclType @@ -2197,7 +2197,7 @@ grantIdentifier : MULT_OPERATOR (DOT_SYMBOL MULT_OPERATOR)? | schemaRef (DOT_SYMBOL MULT_OPERATOR)? | tableRef - | {this.serverVersion >= 80017}? schemaRef DOT_SYMBOL tableRef + | {this.isServerVersionGe80017()}? schemaRef DOT_SYMBOL tableRef ; requireList @@ -2212,7 +2212,7 @@ requireListElement grantOption : option = GRANT_SYMBOL OPTION_SYMBOL - | {this.serverVersion < 80011}? ( + | {this.isServerVersionLt80011()}? ( option = MAX_QUERIES_PER_HOUR_SYMBOL ulong_number | option = MAX_UPDATES_PER_HOUR_SYMBOL ulong_number | option = MAX_CONNECTIONS_PER_HOUR_SYMBOL ulong_number @@ -2249,16 +2249,16 @@ tableAdministrationStatement ; histogramAutoUpdate - : {this.serverVersion >= 80200}? (MANUAL_SYMBOL | AUTO_SYMBOL) UPDATE_SYMBOL + : {this.isServerVersionGe80200()}? (MANUAL_SYMBOL | AUTO_SYMBOL) UPDATE_SYMBOL ; histogramUpdateParam : histogramNumBuckets? histogramAutoUpdate? - | {this.serverVersion >= 80031}? USING_SYMBOL DATA_SYMBOL textStringLiteral + | {this.isServerVersionGe80031()}? USING_SYMBOL DATA_SYMBOL textStringLiteral ; histogramNumBuckets - : {this.serverVersion >= 80200}? WITH_SYMBOL INT_NUMBER BUCKETS_SYMBOL + : {this.isServerVersionGe80200()}? WITH_SYMBOL INT_NUMBER BUCKETS_SYMBOL ; histogram @@ -2308,7 +2308,7 @@ installSetValue ; installSetValueList - : {this.serverVersion >= 80032}? SET_SYMBOL installSetValue ( + : {this.isServerVersionGe80032()}? SET_SYMBOL installSetValue ( COMMA_SYMBOL installSetValue )* ; @@ -2326,9 +2326,9 @@ startOptionValueList | PASSWORD_SYMBOL (FOR_SYMBOL user)? equal ( textString replacePassword? retainCurrentPassword? | textString replacePassword? retainCurrentPassword? - | {this.serverVersion < 80014}? PASSWORD_SYMBOL OPEN_PAR_SYMBOL textString CLOSE_PAR_SYMBOL + | {this.isServerVersionLt80014()}? PASSWORD_SYMBOL OPEN_PAR_SYMBOL textString CLOSE_PAR_SYMBOL ) - | {this.serverVersion >= 80018}? PASSWORD_SYMBOL (FOR_SYMBOL user)? TO_SYMBOL RANDOM_SYMBOL replacePassword? + | {this.isServerVersionGe80018()}? PASSWORD_SYMBOL (FOR_SYMBOL user)? TO_SYMBOL RANDOM_SYMBOL replacePassword? retainCurrentPassword? ; @@ -2361,7 +2361,7 @@ optionValueNoOptionType | NAMES_SYMBOL ( equal expr | charsetName collate? - | {this.serverVersion >= 80011}? DEFAULT_SYMBOL + | {this.isServerVersionGe80011()}? DEFAULT_SYMBOL ) ; @@ -2420,7 +2420,7 @@ showOpenTablesStatement ; showParseTreeStatement - : {this.serverVersion >= 80100}? SHOW_SYMBOL PARSE_TREE_SYMBOL simpleStatement + : {this.isServerVersionGe80100()}? SHOW_SYMBOL PARSE_TREE_SYMBOL simpleStatement ; showPluginsStatement @@ -2775,7 +2775,7 @@ utilityStatement | explainStatement | helpCommand | useCommand - | {this.serverVersion >= 80011}? restartServer + | {this.isServerVersionGe80011()}? restartServer ; describeStatement @@ -2787,17 +2787,17 @@ describeStatement explainStatement : (EXPLAIN_SYMBOL | DESCRIBE_SYMBOL | DESC_SYMBOL) explainOptions? ( - {this.serverVersion >= 80032}? FOR_SYMBOL DATABASE_SYMBOL textOrIdentifier + {this.isServerVersionGe80032()}? FOR_SYMBOL DATABASE_SYMBOL textOrIdentifier )? explainableStatement ; explainOptions : FORMAT_SYMBOL EQUAL_OPERATOR textOrIdentifier ( - {this.serverVersion >= 80032}? explainInto + {this.isServerVersionGe80032()}? explainInto )? - | {this.serverVersion < 80012}? EXTENDED_SYMBOL - | {this.serverVersion >= 80018}? ANALYZE_SYMBOL - | {this.serverVersion >= 80019}? ANALYZE_SYMBOL FORMAT_SYMBOL EQUAL_OPERATOR textOrIdentifier + | {this.isServerVersionLt80012()}? EXTENDED_SYMBOL + | {this.isServerVersionGe80018()}? ANALYZE_SYMBOL + | {this.isServerVersionGe80019()}? ANALYZE_SYMBOL FORMAT_SYMBOL EQUAL_OPERATOR textOrIdentifier ; explainableStatement @@ -2857,7 +2857,7 @@ compOp predicate : bitExpr ( notRule? predicateOperations - | {this.serverVersion >= 80017}? MEMBER_SYMBOL OF_SYMBOL? simpleExprWithParentheses + | {this.isServerVersionGe80017()}? MEMBER_SYMBOL OF_SYMBOL? simpleExprWithParentheses | SOUNDS_SYMBOL LIKE_SYMBOL bitExpr )? ; @@ -2921,7 +2921,7 @@ simpleExpr // $antlr-format groupedAlignments on arrayCast - : {this.serverVersion >= 80017}? ARRAY_SYMBOL + : {this.isServerVersionGe80017()}? ARRAY_SYMBOL ; jsonOperator @@ -2967,7 +2967,7 @@ windowFunctionCall ) parentheses windowingClause | NTILE_SYMBOL ( OPEN_PAR_SYMBOL stableInteger CLOSE_PAR_SYMBOL - | {this.serverVersion < 80024}? simpleExprWithParentheses + | {this.isServerVersionLt80024()}? simpleExprWithParentheses ) windowingClause | (LEAD_SYMBOL | LAG_SYMBOL) OPEN_PAR_SYMBOL expr leadLagInfo? CLOSE_PAR_SYMBOL nullTreatment? windowingClause | (FIRST_VALUE_SYMBOL | LAST_VALUE_SYMBOL) exprWithParentheses nullTreatment? windowingClause @@ -2988,7 +2988,7 @@ samplingPercentage ; tablesampleClause - : {this.serverVersion >= 80200}? TABLESAMPLE_SYMBOL samplingMethod OPEN_PAR_SYMBOL samplingPercentage CLOSE_PAR_SYMBOL + : {this.isServerVersionGe80200()}? TABLESAMPLE_SYMBOL samplingMethod OPEN_PAR_SYMBOL samplingPercentage CLOSE_PAR_SYMBOL ; windowingClause @@ -2999,7 +2999,7 @@ leadLagInfo : COMMA_SYMBOL ( ulonglongNumber | PARAM_MARKER - | {this.serverVersion >= 80024}? stableInteger + | {this.isServerVersionGe80024()}? stableInteger ) (COMMA_SYMBOL expr)? ; @@ -3079,7 +3079,7 @@ runtimeFunctionCall | (DATE_ADD_SYMBOL | DATE_SUB_SYMBOL) OPEN_PAR_SYMBOL expr COMMA_SYMBOL INTERVAL_SYMBOL expr interval CLOSE_PAR_SYMBOL | EXTRACT_SYMBOL OPEN_PAR_SYMBOL interval FROM_SYMBOL expr CLOSE_PAR_SYMBOL | GET_FORMAT_SYMBOL OPEN_PAR_SYMBOL dateTimeTtype COMMA_SYMBOL expr CLOSE_PAR_SYMBOL - | {this.serverVersion >= 80032}? LOG_SYMBOL OPEN_PAR_SYMBOL expr ( + | {this.isServerVersionGe80032()}? LOG_SYMBOL OPEN_PAR_SYMBOL expr ( COMMA_SYMBOL expr )? CLOSE_PAR_SYMBOL | NOW_SYMBOL timeFunctionParameters? @@ -3102,7 +3102,7 @@ runtimeFunctionCall | FORMAT_SYMBOL OPEN_PAR_SYMBOL expr COMMA_SYMBOL expr (COMMA_SYMBOL expr)? CLOSE_PAR_SYMBOL | MICROSECOND_SYMBOL exprWithParentheses | MOD_SYMBOL OPEN_PAR_SYMBOL expr COMMA_SYMBOL expr CLOSE_PAR_SYMBOL - | {this.serverVersion < 80011}? PASSWORD_SYMBOL exprWithParentheses + | {this.isServerVersionLt80011()}? PASSWORD_SYMBOL exprWithParentheses | QUARTER_SYMBOL exprWithParentheses | REPEAT_SYMBOL OPEN_PAR_SYMBOL expr COMMA_SYMBOL expr CLOSE_PAR_SYMBOL | REPLACE_SYMBOL OPEN_PAR_SYMBOL expr COMMA_SYMBOL expr COMMA_SYMBOL expr CLOSE_PAR_SYMBOL @@ -3210,7 +3210,7 @@ lvalueVariable : ( // Check in semantic phase that the first id is not global/local/session/default. identifier dotIdentifier? - | {this.serverVersion >= 80017}? lValueIdentifier dotIdentifier? + | {this.isServerVersionGe80017()}? lValueIdentifier dotIdentifier? ) | DEFAULT_SYMBOL dotIdentifier ; @@ -3238,14 +3238,14 @@ castType | SIGNED_SYMBOL INT_SYMBOL? | UNSIGNED_SYMBOL INT_SYMBOL? | DATE_SYMBOL - | {this.serverVersion >= 80024}? YEAR_SYMBOL + | {this.isServerVersionGe80024()}? YEAR_SYMBOL | TIME_SYMBOL typeDatetimePrecision? | DATETIME_SYMBOL typeDatetimePrecision? | DECIMAL_SYMBOL floatOptions? | JSON_SYMBOL - | {this.serverVersion >= 80017}? realType - | {this.serverVersion >= 80017}? FLOAT_SYMBOL standardFloatOptions? - | {this.serverVersion >= 80027}? ( + | {this.isServerVersionGe80017()}? realType + | {this.isServerVersionGe80017()}? FLOAT_SYMBOL standardFloatOptions? + | {this.isServerVersionGe80027()}? ( POINT_SYMBOL | LINESTRING_SYMBOL | POLYGON_SYMBOL @@ -3563,7 +3563,7 @@ columnDefinition ; checkOrReferences - : {this.serverVersion < 80016}? checkConstraint + : {this.isServerVersionLt80016()}? checkConstraint | references ; @@ -3572,7 +3572,7 @@ checkConstraint ; constraintEnforcement - : { this.serverVersion >= 80017}? NOT_SYMBOL? ENFORCED_SYMBOL + : {this.isServerVersionGe80017()}? NOT_SYMBOL? ENFORCED_SYMBOL ; tableConstraintDef @@ -3602,10 +3602,10 @@ fieldDefinition columnAttribute : NOT_SYMBOL? nullLiteral - | {this.serverVersion >= 80014}? NOT_SYMBOL SECONDARY_SYMBOL + | {this.isServerVersionGe80014()}? NOT_SYMBOL SECONDARY_SYMBOL | value = DEFAULT_SYMBOL ( nowOrSignedLiteral - | {this.serverVersion >= 80013}? exprWithParentheses + | {this.isServerVersionGe80013()}? exprWithParentheses ) | value = ON_SYMBOL UPDATE_SYMBOL NOW_SYMBOL timeFunctionParameters? | value = AUTO_INCREMENT_SYMBOL @@ -3617,11 +3617,11 @@ columnAttribute | value = COLUMN_FORMAT_SYMBOL columnFormat | value = STORAGE_SYMBOL storageMedia | value = SRID_SYMBOL real_ulonglong_number - | {this.serverVersion >= 80017}? constraintName? checkConstraint - | {this.serverVersion >= 80017}? constraintEnforcement - | {this.serverVersion >= 80024}? value = ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute - | {this.serverVersion >= 80024}? value = SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute - | {this.serverVersion >= 80024}? visibility + | {this.isServerVersionGe80017()}? constraintName? checkConstraint + | {this.isServerVersionGe80017()}? constraintEnforcement + | {this.isServerVersionGe80024()}? value = ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? value = SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? visibility ; columnFormat @@ -3687,7 +3687,7 @@ keyListWithExpression keyPartOrExpression : // key_part_with_expression in sql_yacc.yy. keyPart - | {this.serverVersion >= 80013}? exprWithParentheses direction? + | {this.isServerVersionGe80013()}? exprWithParentheses direction? ; indexType @@ -3704,8 +3704,8 @@ commonIndexOption : KEY_BLOCK_SIZE_SYMBOL EQUAL_OPERATOR? ulong_number | COMMENT_SYMBOL textLiteral | visibility - | {this.serverVersion >= 80024}? ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute - | {this.serverVersion >= 80024}? SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute ; visibility @@ -3839,13 +3839,13 @@ functionDatetimePrecision charsetName : textOrIdentifier | BINARY_SYMBOL - | {this.serverVersion < 80011}? DEFAULT_SYMBOL + | {this.isServerVersionLt80011()}? DEFAULT_SYMBOL ; collationName : textOrIdentifier - | {this.serverVersion < 80011}? DEFAULT_SYMBOL - | {this.serverVersion >= 80018}? BINARY_SYMBOL + | {this.isServerVersionLt80011()}? DEFAULT_SYMBOL + | {this.isServerVersionGe80018()}? BINARY_SYMBOL ; createTableOptions @@ -3869,7 +3869,7 @@ createTableOptionsSpaceSeparated createTableOption : // In the order as they appear in the server grammar. option = ENGINE_SYMBOL EQUAL_OPERATOR? engineRef - | {this.serverVersion >= 80014}? option = SECONDARY_ENGINE_SYMBOL equal? ( + | {this.isServerVersionGe80014()}? option = SECONDARY_ENGINE_SYMBOL equal? ( NULL_SYMBOL | textOrIdentifier ) @@ -3911,10 +3911,10 @@ createTableOption | option = STORAGE_SYMBOL (DISK_SYMBOL | MEMORY_SYMBOL) | option = CONNECTION_SYMBOL EQUAL_OPERATOR? textString | option = KEY_BLOCK_SIZE_SYMBOL EQUAL_OPERATOR? ulonglongNumber - | {this.serverVersion >= 80024}? option = START_SYMBOL TRANSACTION_SYMBOL - | {this.serverVersion >= 80024}? option = ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute - | {this.serverVersion >= 80024}? option = SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute - | {this.serverVersion >= 80024}? tsOptionAutoextendSize + | {this.isServerVersionGe80024()}? option = START_SYMBOL TRANSACTION_SYMBOL + | {this.isServerVersionGe80024()}? option = ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? option = SECONDARY_ENGINE_ATTRIBUTE_SYMBOL EQUAL_OPERATOR? jsonAttribute + | {this.isServerVersionGe80024()}? tsOptionAutoextendSize ; ternaryOption @@ -4021,7 +4021,7 @@ ifExistsIdentifier persistedVariableIdentifier : identifier - | {this.serverVersion >= 80032}? ( + | {this.isServerVersionGe80032()}? ( qualifiedIdentifier | DEFAULT_SYMBOL dotIdentifier ) @@ -4468,7 +4468,7 @@ ulonglongNumber real_ulonglong_number : INT_NUMBER - | {this.serverVersion >= 80017}? HEX_NUMBER + | {this.isServerVersionGe80017()}? HEX_NUMBER | ULONGLONG_NUMBER | LONG_NUMBER ; @@ -4481,7 +4481,7 @@ signedLiteral signedLiteralOrNull : signedLiteral - | {this.serverVersion >= 80024}? nullAsLiteral + | {this.isServerVersionGe80024()}? nullAsLiteral ; literal @@ -4495,7 +4495,7 @@ literal literalOrNull : literal - | {this.serverVersion >= 80024}? nullAsLiteral + | {this.isServerVersionGe80024()}? nullAsLiteral ; nullAsLiteral @@ -4521,7 +4521,7 @@ textString textStringHash : textStringLiteral - | {this.serverVersion >= 80017}? HEX_NUMBER + | {this.isServerVersionGe80017()}? HEX_NUMBER ; textLiteral @@ -4657,12 +4657,12 @@ jsonAttribute // the rule `ident_keywords_unambiguous`. If they cause grammar conflicts, try // one of `ident_keywords_ambiguous_...` rules instead. identifierKeyword - : {this.serverVersion < 80017}? ( + : {this.isServerVersionLt80017()}? ( labelKeyword | roleOrIdentifierKeyword | EXECUTE_SYMBOL | SHUTDOWN_SYMBOL // Previously allowed as SP label as well. - | {this.serverVersion >= 80011}? RESTART_SYMBOL + | {this.isServerVersionGe80011()}? RESTART_SYMBOL ) | ( identifierKeywordsUnambiguous @@ -4729,7 +4729,7 @@ identifierKeywordsAmbiguous2Labels // ident_keywords_ambiguous_1_roles_and_labels // ident_keywords_ambiguous_2_labels labelKeyword - : {this.serverVersion < 80017}? ( + : {this.isServerVersionLt80017()}? ( roleOrLabelKeyword | EVENT_SYMBOL | FILE_SYMBOL @@ -5180,7 +5180,7 @@ identifierKeywordsUnambiguous | YEAR_SYMBOL | ZONE_SYMBOL ) - | {this.serverVersion >= 80019}? ( + | {this.isServerVersionGe80019()}? ( ARRAY_SYMBOL | FAILED_LOGIN_ATTEMPTS_SYMBOL | MASTER_COMPRESSION_ALGORITHM_SYMBOL @@ -5197,7 +5197,7 @@ identifierKeywordsUnambiguous | TIMESTAMP_SYMBOL | TIME_SYMBOL ) - | {this.serverVersion >= 80200}? ( + | {this.isServerVersionGe80200()}? ( BULK_SYMBOL | GENERATE_SYMBOL | GTIDS_SYMBOL @@ -5216,7 +5216,7 @@ identifierKeywordsUnambiguous // ident_keywords_ambiguous_1_roles_and_labels // ident_keywords_ambiguous_3_roles roleKeyword - : {this.serverVersion < 80017}? (roleOrLabelKeyword | roleOrIdentifierKeyword) + : {this.isServerVersionLt80017()}? (roleOrLabelKeyword | roleOrIdentifierKeyword) | ( identifierKeywordsUnambiguous | identifierKeywordsAmbiguous2Labels @@ -5655,5 +5655,5 @@ roleOrLabelKeyword ) // Tokens that entered or left this rule in specific versions and are not automatically // handled in the lexer. - | {this.serverVersion >= 80014}? ADMIN_SYMBOL + | {this.isServerVersionGe80014()}? ADMIN_SYMBOL ; From 2f8c5af4ebd28da620b4e9fa0cda7a5faf959dcd Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 17:47:55 -0500 Subject: [PATCH 18/29] Fix CSharp target for target agnostic parser. --- sql/mysql/Oracle/CSharp/MySQLParserBase.cs | 51 ++++++++++++---------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs index b7fc9114e7..a2e0c5a430 100644 --- a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs @@ -28,27 +28,32 @@ protected MySQLParserBase(ITokenStream input, TextWriter output, TextWriter erro * * @returns True if the mode is one of the currently active modes. */ - public bool isSqlModeActive(SqlMode mode) { - return this.sqlModes.Contains(mode); - } - - public bool isPureIdentifier() - { - return this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public bool isTextStringLiteral() - { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public bool isStoredRoutineBody() - { - return serverVersion >= 80032 && supportMle; - } - - public bool isSelectStatementWithInto() - { - return serverVersion >= 80024 && serverVersion < 80031; - } + public bool isSqlModeActive(SqlMode mode) { return this.sqlModes.Contains(mode); } + public bool isPureIdentifier() { return this.isSqlModeActive(SqlMode.AnsiQuotes); } + public bool isTextStringLiteral() { return !this.isSqlModeActive(SqlMode.AnsiQuotes); } + public bool isStoredRoutineBody() { return serverVersion >= 80032 && supportMle; } + public bool isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } + public bool isServerVersionGe80004() { return this.serverVersion >= 80004; } + public bool isServerVersionGe80011() { return this.serverVersion >= 80011; } + public bool isServerVersionGe80013() { return this.serverVersion >= 80013; } + public bool isServerVersionGe80014() { return this.serverVersion >= 80014; } + public bool isServerVersionGe80016() { return this.serverVersion >= 80016; } + public bool isServerVersionGe80017() { return this.serverVersion >= 80017; } + public bool isServerVersionGe80018() { return this.serverVersion >= 80018; } + public bool isServerVersionGe80019() { return this.serverVersion >= 80019; } + public bool isServerVersionGe80024() { return this.serverVersion >= 80024; } + public bool isServerVersionGe80025() { return this.serverVersion >= 80025; } + public bool isServerVersionGe80027() { return this.serverVersion >= 80027; } + public bool isServerVersionGe80031() { return this.serverVersion >= 80031; } + public bool isServerVersionGe80032() { return this.serverVersion >= 80032; } + public bool isServerVersionGe80100() { return this.serverVersion >= 80100; } + public bool isServerVersionGe80200() { return this.serverVersion >= 80200; } + public bool isServerVersionLt80011() { return this.serverVersion < 80011; } + public bool isServerVersionLt80012() { return this.serverVersion < 80012; } + public bool isServerVersionLt80014() { return this.serverVersion < 80014; } + public bool isServerVersionLt80016() { return this.serverVersion < 80016; } + public bool isServerVersionLt80017() { return this.serverVersion < 80017; } + public bool isServerVersionLt80024() { return this.serverVersion < 80024; } + public bool isServerVersionLt80025() { return this.serverVersion < 80025; } + public bool isServerVersionLt80031() { return this.serverVersion < 80031; } } From 94ea90e6de27b94cdb4b2bfe790283b5c94d601f Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 18:04:30 -0500 Subject: [PATCH 19/29] Update Cpp port with target agnostic methods. --- sql/mysql/Oracle/Cpp/MySQLParserBase.cpp | 24 ++++++++++++++++++++++++ sql/mysql/Oracle/Cpp/MySQLParserBase.h | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp b/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp index 8306fae146..bdc2f9a733 100644 --- a/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp +++ b/sql/mysql/Oracle/Cpp/MySQLParserBase.cpp @@ -46,3 +46,27 @@ bool MySQLParserBase::isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } + +bool MySQLParserBase::isServerVersionGe80004() { return this->serverVersion >= 80004; } +bool MySQLParserBase::isServerVersionGe80011() { return this->serverVersion >= 80011; } +bool MySQLParserBase::isServerVersionGe80013() { return this->serverVersion >= 80013; } +bool MySQLParserBase::isServerVersionGe80014() { return this->serverVersion >= 80014; } +bool MySQLParserBase::isServerVersionGe80016() { return this->serverVersion >= 80016; } +bool MySQLParserBase::isServerVersionGe80017() { return this->serverVersion >= 80017; } +bool MySQLParserBase::isServerVersionGe80018() { return this->serverVersion >= 80018; } +bool MySQLParserBase::isServerVersionGe80019() { return this->serverVersion >= 80019; } +bool MySQLParserBase::isServerVersionGe80024() { return this->serverVersion >= 80024; } +bool MySQLParserBase::isServerVersionGe80025() { return this->serverVersion >= 80025; } +bool MySQLParserBase::isServerVersionGe80027() { return this->serverVersion >= 80027; } +bool MySQLParserBase::isServerVersionGe80031() { return this->serverVersion >= 80031; } +bool MySQLParserBase::isServerVersionGe80032() { return this->serverVersion >= 80032; } +bool MySQLParserBase::isServerVersionGe80100() { return this->serverVersion >= 80100; } +bool MySQLParserBase::isServerVersionGe80200() { return this->serverVersion >= 80200; } +bool MySQLParserBase::isServerVersionLt80011() { return this->serverVersion < 80011; } +bool MySQLParserBase::isServerVersionLt80012() { return this->serverVersion < 80012; } +bool MySQLParserBase::isServerVersionLt80014() { return this->serverVersion < 80014; } +bool MySQLParserBase::isServerVersionLt80016() { return this->serverVersion < 80016; } +bool MySQLParserBase::isServerVersionLt80017() { return this->serverVersion < 80017; } +bool MySQLParserBase::isServerVersionLt80024() { return this->serverVersion < 80024; } +bool MySQLParserBase::isServerVersionLt80025() { return this->serverVersion < 80025; } +bool MySQLParserBase::isServerVersionLt80031() { return this->serverVersion < 80031; } diff --git a/sql/mysql/Oracle/Cpp/MySQLParserBase.h b/sql/mysql/Oracle/Cpp/MySQLParserBase.h index 2fdc73e419..b642d1639d 100644 --- a/sql/mysql/Oracle/Cpp/MySQLParserBase.h +++ b/sql/mysql/Oracle/Cpp/MySQLParserBase.h @@ -39,4 +39,27 @@ class MySQLParserBase : public antlr4::Parser { bool isTextStringLiteral(); bool isStoredRoutineBody(); bool isSelectStatementWithInto(); + bool isServerVersionGe80004(); + bool isServerVersionGe80011(); + bool isServerVersionGe80013(); + bool isServerVersionGe80014(); + bool isServerVersionGe80016(); + bool isServerVersionGe80017(); + bool isServerVersionGe80018(); + bool isServerVersionGe80019(); + bool isServerVersionGe80024(); + bool isServerVersionGe80025(); + bool isServerVersionGe80027(); + bool isServerVersionGe80031(); + bool isServerVersionGe80032(); + bool isServerVersionGe80100(); + bool isServerVersionGe80200(); + bool isServerVersionLt80011(); + bool isServerVersionLt80012(); + bool isServerVersionLt80014(); + bool isServerVersionLt80016(); + bool isServerVersionLt80017(); + bool isServerVersionLt80024(); + bool isServerVersionLt80025(); + bool isServerVersionLt80031(); }; From 4baed1984f7d986b00a820e44722d07ec153e949 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 18:11:01 -0500 Subject: [PATCH 20/29] Clean up CSharp port code. --- sql/mysql/Oracle/CSharp/MySQLParserBase.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs index a2e0c5a430..f64d85473e 100644 --- a/sql/mysql/Oracle/CSharp/MySQLParserBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLParserBase.cs @@ -21,13 +21,6 @@ protected MySQLParserBase(ITokenStream input, TextWriter output, TextWriter erro this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } - /** - * Determines if the given SQL mode is currently active in the lexer. - * - * @param mode The mode to check. - * - * @returns True if the mode is one of the currently active modes. - */ public bool isSqlModeActive(SqlMode mode) { return this.sqlModes.Contains(mode); } public bool isPureIdentifier() { return this.isSqlModeActive(SqlMode.AnsiQuotes); } public bool isTextStringLiteral() { return !this.isSqlModeActive(SqlMode.AnsiQuotes); } From e8ab64e60fa873005c4ebc38f73d8c1fd114b724 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 18:45:37 -0500 Subject: [PATCH 21/29] CSharp token props now perfect. --- sql/mysql/Oracle/CSharp/MySQLLexerBase.cs | 40 +++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs index f2524e14e2..176a06a25d 100644 --- a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs @@ -16,7 +16,9 @@ public class MySQLLexerBase : Lexer { public HashSet sqlModes = new HashSet(); /** Enable Multi Language Extension support. */ - public bool supportMle = true; + public bool supportMle = true; + + bool justEmittedDot = false; public HashSet charSets = new HashSet(); // Used to check repertoires. protected bool inVersionComment = false; @@ -33,8 +35,6 @@ public class MySQLLexerBase : Lexer { static string unsignedLongLongString = "18446744073709551615"; static int unsignedLongLongLength = 20; - private bool JustEmitedDot = false; - public override string[] RuleNames => throw new NotImplementedException(); public override IVocabulary Vocabulary => throw new NotImplementedException(); @@ -254,15 +254,28 @@ protected int checkCharset(string text) */ protected void emitDot() { - this.pendingTokens.Enqueue(this.TokenFactory.Create(new Tuple(this, (ICharStream)this.InputStream), MySQLLexer.DOT_SYMBOL, - this.Text, this.Channel, this.TokenStartCharIndex, this.TokenStartCharIndex, this.Line, - this.Column - )); - + var len = this.Text.Length; + var t = this.TokenFactory.Create(new Tuple(this, (ICharStream)this.InputStream), MySQLLexer.DOT_SYMBOL, + ".", this.Channel, this.TokenStartCharIndex, this.TokenStartCharIndex, this.Line, this.Column) as CommonToken; + this.pendingTokens.Enqueue(t); + t.Column = t.Column - len; ++this.Column; - this.JustEmitedDot = true; + this.justEmittedDot = true; } + public override IToken Emit() + { + var t = base.Emit(); + if (this.justEmittedDot) { + var p = t as CommonToken; + p.Text = p.Text.Substring(1); + p.Column = p.Column + 1; + p.StartIndex = p.StartIndex + 1; + this.Column = this.Column - 1; + this.justEmittedDot = false; + } + return t; + } // Version-related methods public bool isServerVersionLt80024() => serverVersion < 80024; @@ -343,15 +356,6 @@ public bool isSingleQuotedText() return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } - public override IToken Emit() - { - IToken t = this.TokenFactory.Create(new Tuple(this, (ICharStream)this.InputStream), - this.Type, (this.Text!=null?(this.JustEmitedDot?this.Text.Substring(1):this.Text):null), this.Channel, this.TokenStartCharIndex + (this.JustEmitedDot?1:0), CharIndex - 1, this.TokenStartLine, this.TokenStartColumn); - this.JustEmitedDot = false; - base.Emit(t); - return t; - } - public void startInVersionComment() { inVersionComment = true; From 49b970931e78a9e650c05402ecb1c6583c804654 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 19:28:24 -0500 Subject: [PATCH 22/29] Fix formatting. --- sql/mysql/Oracle/CSharp/MySQLLexerBase.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs index 176a06a25d..a7f7a03771 100644 --- a/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs +++ b/sql/mysql/Oracle/CSharp/MySQLLexerBase.cs @@ -16,9 +16,9 @@ public class MySQLLexerBase : Lexer { public HashSet sqlModes = new HashSet(); /** Enable Multi Language Extension support. */ - public bool supportMle = true; + public bool supportMle = true; - bool justEmittedDot = false; + bool justEmittedDot = false; public HashSet charSets = new HashSet(); // Used to check repertoires. protected bool inVersionComment = false; @@ -266,10 +266,10 @@ protected void emitDot() public override IToken Emit() { var t = base.Emit(); - if (this.justEmittedDot) { - var p = t as CommonToken; - p.Text = p.Text.Substring(1); - p.Column = p.Column + 1; + if (this.justEmittedDot) { + var p = t as CommonToken; + p.Text = p.Text.Substring(1); + p.Column = p.Column + 1; p.StartIndex = p.StartIndex + 1; this.Column = this.Column - 1; this.justEmittedDot = false; From 12b82368e6fef9fd5f9877e1f938c07d3d1008d1 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 19:28:58 -0500 Subject: [PATCH 23/29] Fix Dart port. --- sql/mysql/Oracle/Dart/MySQLLexerBase.dart | 94 ++++++---------------- sql/mysql/Oracle/Dart/MySQLParserBase.dart | 52 ++++++------ 2 files changed, 53 insertions(+), 93 deletions(-) diff --git a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart index 93d37abcb3..ae5f8b4e82 100644 --- a/sql/mysql/Oracle/Dart/MySQLLexerBase.dart +++ b/sql/mysql/Oracle/Dart/MySQLLexerBase.dart @@ -28,7 +28,7 @@ abstract class MySQLLexerBase extends Lexer static const String unsignedLongLongString = "18446744073709551615"; static const int unsignedLongLongLength = 20; - bool justEmitedDot = false; + bool justEmittedDot = false; MySQLLexerBase(CharStream input) : super(input) { @@ -223,23 +223,35 @@ abstract class MySQLLexerBase extends Lexer */ void emitDot() { + var len = this.text.length; var ctf = this.tokenFactory; Token t = ctf.create( MySQLLexer.TOKEN_DOT_SYMBOL, - this.text, + ".", Pair(this, this.inputStream), this.channel, this.tokenStartCharIndex, this.tokenStartCharIndex, this.line, - this.charPositionInLine); + this.charPositionInLine - len); this.pendingTokens.add(t); ++this.charPositionInLine; ++this.tokenStartCharIndex; - this.justEmitedDot = true; + this.justEmittedDot = true; + } + + @override Token emit() + { + var t = super.emit(); + if (this.justEmittedDot) { + var p = t as CommonToken; + p.charPositionInLine = p.charPositionInLine + 1; + this.charPositionInLine = this.charPositionInLine - 1; + this.justEmittedDot = false; + } + return t; } - // Version-related methods bool isServerVersionLt80024() => serverVersion < 80024; bool isServerVersionGe80024() => serverVersion >= 80024; bool isServerVersionGe80011() => serverVersion >= 80011; @@ -248,22 +260,10 @@ abstract class MySQLLexerBase extends Lexer bool isServerVersionGe80014() => serverVersion >= 80014; bool isServerVersionGe80017() => serverVersion >= 80017; bool isServerVersionGe80018() => serverVersion >= 80018; - bool isMasterCompressionAlgorithm() => serverVersion >= 80018 && isServerVersionLt80024(); - bool isServerVersionLt80031() => serverVersion < 80031; - - // Functions for specific token types - void doLogicalOr() - { - this.type = isSqlModeActive(SqlMode.pipesAsConcat) ? MySQLLexer.TOKEN_CONCAT_PIPES_SYMBOL : MySQLLexer.TOKEN_LOGICAL_OR_OPERATOR; - } - - void doIntNumber() - { - this.type = determineNumericType(this.text); - } - + void doLogicalOr() { this.type = isSqlModeActive(SqlMode.pipesAsConcat) ? MySQLLexer.TOKEN_CONCAT_PIPES_SYMBOL : MySQLLexer.TOKEN_LOGICAL_OR_OPERATOR; } + void doIntNumber() { this.type = determineNumericType(this.text); } void doAdddate() => this.type = determineFunction(MySQLLexer.TOKEN_ADDDATE_SYMBOL); void doBitAnd() => this.type = determineFunction(MySQLLexer.TOKEN_BIT_AND_SYMBOL); void doBitOr() => this.type = determineFunction(MySQLLexer.TOKEN_BIT_OR_SYMBOL); @@ -300,55 +300,11 @@ abstract class MySQLLexerBase extends Lexer void doVarPop() => this.type = determineFunction(MySQLLexer.TOKEN_VARIANCE_SYMBOL); void doVarSamp() => this.type = determineFunction(MySQLLexer.TOKEN_VAR_SAMP_SYMBOL); void doUnderscoreCharset() => this.type = checkCharset(this.text); - bool isVersionComment() => checkMySQLVersion(this.text); - - bool isBackTickQuotedId() - { - return !this.isSqlModeActive(SqlMode.noBackslashEscapes); - } - - bool isDoubleQuotedText() - { - return !this.isSqlModeActive(SqlMode.noBackslashEscapes); - } - - bool isSingleQuotedText() - { - return !this.isSqlModeActive(SqlMode.noBackslashEscapes); - } - -/* - @override Token emit() - { - var ctf = this.tokenFactory; - Token t = ctf.create( - this.type, - (this.text!=null?(this.justEmitedDot?this.text.substring(1):this.text):null), - Pair(this, this.inputStream), - this.channel, - this.tokenStartCharIndex + (this.justEmitedDot?1:0), - CharIndex - 1, - this.line, - this.charPositionInLine); - this.justEmitedDot = false; - super.emit(t); - return t; - } -*/ - - void startInVersionComment() - { - inVersionComment = true; - } - - void endInVersionComment() - { - inVersionComment = false; - } - - bool isInVersionComment() - { - return inVersionComment; - } + bool isBackTickQuotedId() { return !this.isSqlModeActive(SqlMode.noBackslashEscapes); } + bool isDoubleQuotedText() { return !this.isSqlModeActive(SqlMode.noBackslashEscapes); } + bool isSingleQuotedText() { return !this.isSqlModeActive(SqlMode.noBackslashEscapes); } + void startInVersionComment() { inVersionComment = true; } + void endInVersionComment() { inVersionComment = false; } + bool isInVersionComment() { return inVersionComment; } } diff --git a/sql/mysql/Oracle/Dart/MySQLParserBase.dart b/sql/mysql/Oracle/Dart/MySQLParserBase.dart index d9749f002e..d1a9721918 100644 --- a/sql/mysql/Oracle/Dart/MySQLParserBase.dart +++ b/sql/mysql/Oracle/Dart/MySQLParserBase.dart @@ -22,28 +22,32 @@ abstract class MySQLParserBase extends Parser { this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } - /// Determines if the given SQL mode is currently active. - bool isSqlModeActive(SqlMode mode) { - return sqlModes.contains(mode); - } - - /// Checks if the parser is in pure identifier mode. - bool isPureIdentifier() { - return isSqlModeActive(SqlMode.ansiQuotes); - } - - /// Checks if the parser is in text string literal mode. - bool isTextStringLiteral() { - return !isSqlModeActive(SqlMode.ansiQuotes); - } - - /// Checks if the parser is handling a stored routine body. - bool isStoredRoutineBody() { - return serverVersion >= 80032 && supportMle; - } - - /// Checks if the parser is handling a SELECT statement with INTO clause. - bool isSelectStatementWithInto() { - return serverVersion >= 80024 && serverVersion < 80031; - } + bool isSqlModeActive(SqlMode mode) { return sqlModes.contains(mode); } + bool isPureIdentifier() { return isSqlModeActive(SqlMode.ansiQuotes); } + bool isTextStringLiteral() { return !isSqlModeActive(SqlMode.ansiQuotes); } + bool isStoredRoutineBody() { return serverVersion >= 80032 && supportMle; } + bool isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } + bool isServerVersionGe80004() { return this.serverVersion >= 80004; } + bool isServerVersionGe80011() { return this.serverVersion >= 80011; } + bool isServerVersionGe80013() { return this.serverVersion >= 80013; } + bool isServerVersionGe80014() { return this.serverVersion >= 80014; } + bool isServerVersionGe80016() { return this.serverVersion >= 80016; } + bool isServerVersionGe80017() { return this.serverVersion >= 80017; } + bool isServerVersionGe80018() { return this.serverVersion >= 80018; } + bool isServerVersionGe80019() { return this.serverVersion >= 80019; } + bool isServerVersionGe80024() { return this.serverVersion >= 80024; } + bool isServerVersionGe80025() { return this.serverVersion >= 80025; } + bool isServerVersionGe80027() { return this.serverVersion >= 80027; } + bool isServerVersionGe80031() { return this.serverVersion >= 80031; } + bool isServerVersionGe80032() { return this.serverVersion >= 80032; } + bool isServerVersionGe80100() { return this.serverVersion >= 80100; } + bool isServerVersionGe80200() { return this.serverVersion >= 80200; } + bool isServerVersionLt80011() { return this.serverVersion < 80011; } + bool isServerVersionLt80012() { return this.serverVersion < 80012; } + bool isServerVersionLt80014() { return this.serverVersion < 80014; } + bool isServerVersionLt80016() { return this.serverVersion < 80016; } + bool isServerVersionLt80017() { return this.serverVersion < 80017; } + bool isServerVersionLt80024() { return this.serverVersion < 80024; } + bool isServerVersionLt80025() { return this.serverVersion < 80025; } + bool isServerVersionLt80031() { return this.serverVersion < 80031; } } From 785694741a1845b7b6e7de44101dd949e87ba6be Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 19:50:59 -0500 Subject: [PATCH 24/29] Fixed Go port, now working. --- sql/mysql/Oracle/Go/MySQLLexerBase.go | 41 +++++++++++++++----------- sql/mysql/Oracle/Go/MySQLParserBase.go | 27 +++++++++++++++-- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/sql/mysql/Oracle/Go/MySQLLexerBase.go b/sql/mysql/Oracle/Go/MySQLLexerBase.go index a98e5b8f33..0e7960d453 100644 --- a/sql/mysql/Oracle/Go/MySQLLexerBase.go +++ b/sql/mysql/Oracle/Go/MySQLLexerBase.go @@ -68,40 +68,45 @@ func NewMySQLLexerBase(input antlr.CharStream) *MySQLLexerBase { } func (l *MySQLLexerBase) MakeCommonToken(ttype int, text string) antlr.Token { - stop := l.TokenStartCharIndex - 1 - start := stop - if len(text) != 0 { - start = stop - len(text) + 1 - } ctf := l.GetTokenFactory() t := ctf.Create( l.GetTokenSourceCharStreamPair(), ttype, text, antlr.TokenDefaultChannel, - start, - l.TokenStartCharIndex-1, + l.TokenStartCharIndex, + l.TokenStartCharIndex, l.TokenStartLine, l.TokenStartColumn) return t } func (m *MySQLLexerBase) emitDot() { - m.pendingTokens = append(m.pendingTokens, m.MakeCommonToken(MySQLLexerDOT_SYMBOL, m.GetText())) + ctf := m.GetTokenFactory() + t := ctf.Create( + m.GetTokenSourceCharStreamPair(), + MySQLLexerDOT_SYMBOL, + ".", + antlr.TokenDefaultChannel, + m.TokenStartCharIndex, + m.TokenStartCharIndex, + m.Interpreter.GetLine(), + m.Interpreter.GetCharPositionInLine() - len(m.GetText())) + m.pendingTokens = append(m.pendingTokens, t) m.TokenStartColumn = m.TokenStartColumn + 1 m.TokenStartCharIndex = m.TokenStartCharIndex + 1 } -func (m *MySQLLexerBase) isServerVersionLt80024() bool { return m.serverVersion < 80024 } -func (m *MySQLLexerBase) isServerVersionGe80024() bool { return m.serverVersion >= 80024 } -func (m *MySQLLexerBase) isServerVersionGe80011() bool { return m.serverVersion >= 80011 } -func (m *MySQLLexerBase) isServerVersionGe80013() bool { return m.serverVersion >= 80013 } -func (m *MySQLLexerBase) isServerVersionLt80014() bool { return m.serverVersion < 80014 } -func (m *MySQLLexerBase) isServerVersionGe80014() bool { return m.serverVersion >= 80014 } -func (m *MySQLLexerBase) isServerVersionGe80017() bool { return m.serverVersion >= 80017 } -func (m *MySQLLexerBase) isServerVersionGe80018() bool { return m.serverVersion >= 80018 } -func (m *MySQLLexerBase) isMasterCompressionAlgorithm() bool { return m.serverVersion >= 80018 && m.isServerVersionLt80024() } -func (m *MySQLLexerBase) isServerVersionLt80031() bool { return m.serverVersion < 80031 } +func (m *MySQLLexerBase) isServerVersionLt80024() bool { return StaticMySQLLexerBase.serverVersion < 80024 } +func (m *MySQLLexerBase) isServerVersionGe80024() bool { return StaticMySQLLexerBase.serverVersion >= 80024 } +func (m *MySQLLexerBase) isServerVersionGe80011() bool { return StaticMySQLLexerBase.serverVersion >= 80011 } +func (m *MySQLLexerBase) isServerVersionGe80013() bool { return StaticMySQLLexerBase.serverVersion >= 80013 } +func (m *MySQLLexerBase) isServerVersionLt80014() bool { return StaticMySQLLexerBase.serverVersion < 80014 } +func (m *MySQLLexerBase) isServerVersionGe80014() bool { return StaticMySQLLexerBase.serverVersion >= 80014 } +func (m *MySQLLexerBase) isServerVersionGe80017() bool { return StaticMySQLLexerBase.serverVersion >= 80017 } +func (m *MySQLLexerBase) isServerVersionGe80018() bool { return StaticMySQLLexerBase.serverVersion >= 80018 } +func (m *MySQLLexerBase) isMasterCompressionAlgorithm() bool { return StaticMySQLLexerBase.serverVersion >= 80018 && m.isServerVersionLt80024() } +func (m *MySQLLexerBase) isServerVersionLt80031() bool { return StaticMySQLLexerBase.serverVersion < 80031 } func (m *MySQLLexerBase) doLogicalOr() { if m.isSqlModeActive(PipesAsConcat) { diff --git a/sql/mysql/Oracle/Go/MySQLParserBase.go b/sql/mysql/Oracle/Go/MySQLParserBase.go index 268549f0fb..9322abb875 100644 --- a/sql/mysql/Oracle/Go/MySQLParserBase.go +++ b/sql/mysql/Oracle/Go/MySQLParserBase.go @@ -47,11 +47,34 @@ func (m *MySQLParserBase) isTextStringLiteral() bool { // isStoredRoutineBody checks if the server version supports stored routine body. func (m *MySQLParserBase) isStoredRoutineBody() bool { - return m.serverVersion >= 80032 && m.supportMle + return StaticMySQLParserBase.serverVersion >= 80032 && StaticMySQLParserBase.supportMle } // isSelectStatementWithInto checks if the server version supports SELECT INTO syntax. func (m *MySQLParserBase) isSelectStatementWithInto() bool { - return m.serverVersion >= 80024 && m.serverVersion < 80031 + return StaticMySQLParserBase.serverVersion >= 80024 && StaticMySQLParserBase.serverVersion < 80031 } +func (m *MySQLParserBase) isServerVersionGe80004() bool { return StaticMySQLParserBase.serverVersion >= 80004 } +func (m *MySQLParserBase) isServerVersionGe80011() bool { return StaticMySQLParserBase.serverVersion >= 80011 } +func (m *MySQLParserBase) isServerVersionGe80013() bool { return StaticMySQLParserBase.serverVersion >= 80013 } +func (m *MySQLParserBase) isServerVersionGe80014() bool { return StaticMySQLParserBase.serverVersion >= 80014 } +func (m *MySQLParserBase) isServerVersionGe80016() bool { return StaticMySQLParserBase.serverVersion >= 80016 } +func (m *MySQLParserBase) isServerVersionGe80017() bool { return StaticMySQLParserBase.serverVersion >= 80017 } +func (m *MySQLParserBase) isServerVersionGe80018() bool { return StaticMySQLParserBase.serverVersion >= 80018 } +func (m *MySQLParserBase) isServerVersionGe80019() bool { return StaticMySQLParserBase.serverVersion >= 80019 } +func (m *MySQLParserBase) isServerVersionGe80024() bool { return StaticMySQLParserBase.serverVersion >= 80024 } +func (m *MySQLParserBase) isServerVersionGe80025() bool { return StaticMySQLParserBase.serverVersion >= 80025 } +func (m *MySQLParserBase) isServerVersionGe80027() bool { return StaticMySQLParserBase.serverVersion >= 80027 } +func (m *MySQLParserBase) isServerVersionGe80031() bool { return StaticMySQLParserBase.serverVersion >= 80031 } +func (m *MySQLParserBase) isServerVersionGe80032() bool { return StaticMySQLParserBase.serverVersion >= 80032 } +func (m *MySQLParserBase) isServerVersionGe80100() bool { return StaticMySQLParserBase.serverVersion >= 80100 } +func (m *MySQLParserBase) isServerVersionGe80200() bool { return StaticMySQLParserBase.serverVersion >= 80200 } +func (m *MySQLParserBase) isServerVersionLt80011() bool { return StaticMySQLParserBase.serverVersion < 80011 } +func (m *MySQLParserBase) isServerVersionLt80012() bool { return StaticMySQLParserBase.serverVersion < 80012 } +func (m *MySQLParserBase) isServerVersionLt80014() bool { return StaticMySQLParserBase.serverVersion < 80014 } +func (m *MySQLParserBase) isServerVersionLt80016() bool { return StaticMySQLParserBase.serverVersion < 80016 } +func (m *MySQLParserBase) isServerVersionLt80017() bool { return StaticMySQLParserBase.serverVersion < 80017 } +func (m *MySQLParserBase) isServerVersionLt80024() bool { return StaticMySQLParserBase.serverVersion < 80024 } +func (m *MySQLParserBase) isServerVersionLt80025() bool { return StaticMySQLParserBase.serverVersion < 80025 } +func (m *MySQLParserBase) isServerVersionLt80031() bool { return StaticMySQLParserBase.serverVersion < 80031 } From 6872b8261a4adc2e2061792767c84f278c4532e1 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 20:18:01 -0500 Subject: [PATCH 25/29] Fixed Java port. --- sql/mysql/Oracle/Java/MySQLLexerBase.java | 99 +++++++--------------- sql/mysql/Oracle/Java/MySQLParserBase.java | 57 ++++++------- 2 files changed, 58 insertions(+), 98 deletions(-) diff --git a/sql/mysql/Oracle/Java/MySQLLexerBase.java b/sql/mysql/Oracle/Java/MySQLLexerBase.java index 44a3959ced..c9b9963097 100644 --- a/sql/mysql/Oracle/Java/MySQLLexerBase.java +++ b/sql/mysql/Oracle/Java/MySQLLexerBase.java @@ -13,9 +13,9 @@ public abstract class MySQLLexerBase extends Lexer { public MySQLLexerBase(CharStream input) { - super(input); - this.serverVersion = 80200; - this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); + super(input); + this.serverVersion = 80200; + this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } public int serverVersion = 0; @@ -39,18 +39,9 @@ public MySQLLexerBase(CharStream input) { static String unsignedLongLongString = "18446744073709551615"; static int unsignedLongLongLength = 20; - private boolean justEmitedDot = false; - - /** - * Determines if the given SQL mode is currently active in the lexer. - * - * @param mode The mode to check. - * - * @returns True if the mode is one of the currently active modes. - */ - public boolean isSqlModeActive(SqlMode mode) { - return this.sqlModes.contains(mode); - } + private boolean justEmittedDot = false; + + public boolean isSqlModeActive(SqlMode mode) { return this.sqlModes.contains(mode); } @Override public void reset() { @@ -178,10 +169,23 @@ protected int checkCharset(String text) { } protected void emitDot() { + var len = this.getText().length(); pendingTokens.add(this._factory.create(this._tokenFactorySourcePair, MySQLLexer.DOT_SYMBOL, - this.getText(), this._channel, this._tokenStartCharIndex, this._tokenStartCharIndex, this.getLine(), this.getCharPositionInLine())); + ".", this._channel, this._tokenStartCharIndex, this._tokenStartCharIndex, this.getLine(), this.getCharPositionInLine() - len)); ++this._tokenStartCharPositionInLine; - this.justEmitedDot = true; + this.justEmittedDot = true; + } + + @Override + public Token emit() { + var t = super.emit(); + if (this.justEmittedDot) { + var p = (CommonToken)t; + p.setText(p.getText().substring(1)); + p.setStartIndex(p.getStartIndex() + 1); + this.justEmittedDot = false; + } + return t; } public boolean isServerVersionLt80024() { return serverVersion < 80024; } @@ -194,15 +198,8 @@ protected void emitDot() { public boolean isServerVersionGe80018() { return serverVersion >= 80018; } public boolean isMasterCompressionAlgorithm() { return serverVersion >= 80018 && isServerVersionLt80024(); } public boolean isServerVersionLt80031() { return serverVersion < 80031; } - - public void doLogicalOr() { - this._type = isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; - } - - public void doIntNumber() { - this._type = determineNumericType(this.getText()); - } - + public void doLogicalOr() { this._type = isSqlModeActive(SqlMode.PipesAsConcat) ? MySQLLexer.CONCAT_PIPES_SYMBOL : MySQLLexer.LOGICAL_OR_OPERATOR; } + public void doIntNumber() { this._type = determineNumericType(this.getText()); } public void doAdddate() { this._type = determineFunction(MySQLLexer.ADDDATE_SYMBOL); } public void doBitAnd() { this._type = determineFunction(MySQLLexer.BIT_AND_SYMBOL); } public void doBitOr() { this._type = determineFunction(MySQLLexer.BIT_OR_SYMBOL); } @@ -239,47 +236,11 @@ public void doIntNumber() { public void doVarPop() { this._type = determineFunction(MySQLLexer.VARIANCE_SYMBOL); } public void doVarSamp() { this._type = determineFunction(MySQLLexer.VAR_SAMP_SYMBOL); } public void doUnderscoreCharset() { this._type = checkCharset(this.getText()); } - - public boolean isVersionComment() { - return checkMySQLVersion(this.getText()); - } - - public boolean isBackTickQuotedId() - { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); - } - - public boolean isDoubleQuotedText() - { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); - } - - public boolean isSingleQuotedText() - { - return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); - } - - @Override - public Token emit() { - Token t = this._factory.create(this._tokenFactorySourcePair, - this._type, this.getText() != null ? (this.justEmitedDot ? this.getText().substring(1) : this.getText()) : null, this._channel, this._tokenStartCharIndex + (this.justEmitedDot ? 1 : 0), this.getCharIndex() - 1, this.getLine(), this.getCharPositionInLine()); - this.justEmitedDot = false; - super.emit(t); - return t; - } - - public void startInVersionComment() - { - inVersionComment = true; - } - - public void endInVersionComment() - { - inVersionComment = false; - } - - public boolean isInVersionComment() - { - return inVersionComment; - } + public boolean isVersionComment() { return checkMySQLVersion(this.getText()); } + public boolean isBackTickQuotedId() { return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } + public boolean isDoubleQuotedText() { return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } + public boolean isSingleQuotedText() { return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } + public void startInVersionComment() { inVersionComment = true; } + public void endInVersionComment() { inVersionComment = false; } + public boolean isInVersionComment() { return inVersionComment; } } diff --git a/sql/mysql/Oracle/Java/MySQLParserBase.java b/sql/mysql/Oracle/Java/MySQLParserBase.java index 352bbe233b..3ba6a9f6a0 100644 --- a/sql/mysql/Oracle/Java/MySQLParserBase.java +++ b/sql/mysql/Oracle/Java/MySQLParserBase.java @@ -21,34 +21,33 @@ protected MySQLParserBase(TokenStream input) { this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } - /** - * Determines if the given SQL mode is currently active in the lexer. - * - * @param mode The mode to check. - * - * @returns True if the mode is one of the currently active modes. - */ - public boolean isSqlModeActive(SqlMode mode) { - return this.sqlModes.contains(mode); - } - - public boolean isPureIdentifier() - { - return this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public boolean isTextStringLiteral() - { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public boolean isStoredRoutineBody() - { - return serverVersion >= 80032 && supportMle; - } + public boolean isSqlModeActive(SqlMode mode) { return this.sqlModes.contains(mode); } + public boolean isPureIdentifier() { return this.isSqlModeActive(SqlMode.AnsiQuotes); } + public boolean isTextStringLiteral() { return !this.isSqlModeActive(SqlMode.AnsiQuotes); } + public boolean isStoredRoutineBody() { return serverVersion >= 80032 && supportMle; } + public boolean isSelectStatementWithInto() { return serverVersion >= 80024 && serverVersion < 80031; } + public boolean isServerVersionGe80004() { return this.serverVersion >= 80004; } + public boolean isServerVersionGe80011() { return this.serverVersion >= 80011; } + public boolean isServerVersionGe80013() { return this.serverVersion >= 80013; } + public boolean isServerVersionGe80014() { return this.serverVersion >= 80014; } + public boolean isServerVersionGe80016() { return this.serverVersion >= 80016; } + public boolean isServerVersionGe80017() { return this.serverVersion >= 80017; } + public boolean isServerVersionGe80018() { return this.serverVersion >= 80018; } + public boolean isServerVersionGe80019() { return this.serverVersion >= 80019; } + public boolean isServerVersionGe80024() { return this.serverVersion >= 80024; } + public boolean isServerVersionGe80025() { return this.serverVersion >= 80025; } + public boolean isServerVersionGe80027() { return this.serverVersion >= 80027; } + public boolean isServerVersionGe80031() { return this.serverVersion >= 80031; } + public boolean isServerVersionGe80032() { return this.serverVersion >= 80032; } + public boolean isServerVersionGe80100() { return this.serverVersion >= 80100; } + public boolean isServerVersionGe80200() { return this.serverVersion >= 80200; } + public boolean isServerVersionLt80011() { return this.serverVersion < 80011; } + public boolean isServerVersionLt80012() { return this.serverVersion < 80012; } + public boolean isServerVersionLt80014() { return this.serverVersion < 80014; } + public boolean isServerVersionLt80016() { return this.serverVersion < 80016; } + public boolean isServerVersionLt80017() { return this.serverVersion < 80017; } + public boolean isServerVersionLt80024() { return this.serverVersion < 80024; } + public boolean isServerVersionLt80025() { return this.serverVersion < 80025; } + public boolean isServerVersionLt80031() { return this.serverVersion < 80031; } - public boolean isSelectStatementWithInto() - { - return serverVersion >= 80024 && serverVersion < 80031; - } } From af5d69b877436458b9cfdd54dd25b83bb0721759 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 21:08:32 -0500 Subject: [PATCH 26/29] Fixed TypeScript and JavaScript ports. --- sql/mysql/Oracle/JavaScript/MySQLLexerBase.js | 20 ++++--- .../Oracle/JavaScript/MySQLParserBase.js | 50 +++++++++------- sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts | 26 ++++++--- .../Oracle/TypeScript/MySQLParserBase.ts | 58 +++++++++---------- 4 files changed, 86 insertions(+), 68 deletions(-) diff --git a/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js b/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js index 8774e52bb6..1163286ee3 100644 --- a/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js +++ b/sql/mysql/Oracle/JavaScript/MySQLLexerBase.js @@ -22,6 +22,7 @@ class MySQLLexerBase extends Lexer { this.sqlModes = new Set(); /** Enable Multi Language Extension support. */ this.supportMle = true; + this.justEmittedDot = false; this.charSets = new Set(); // Used to check repertoires. this.inVersionComment = false; this.pendingTokens = []; @@ -204,10 +205,21 @@ class MySQLLexerBase extends Lexer { * Creates a DOT token in the token stream. */ emitDot() { + let len = this.text.length; let t = new CommonToken([this, this._input], MySQLLexer.DOT_SYMBOL, 0, this._tokenStartCharIndex, this._tokenStartCharIndex); this.pendingTokens.push(t); - ++this.column; + t.text = "."; + t.column = t.column - len; ++this._tokenStartCharIndex; + this.justEmittedDot = true; + } + emit() { + let t = super.emit(); + if (this.justEmittedDot) { + t.column = t.column + 1; + this.justEmittedDot = false; + } + return t; } isServerVersionLt80024() { return this.serverVersion < 80024; @@ -361,12 +373,6 @@ class MySQLLexerBase extends Lexer { isSingleQuotedText() { return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } - emit() { - let t = super.emit(); - if (t.type == MySQLLexer.WHITESPACE) - t.channel = Token.HIDDEN_CHANNEL; - return t; - } startInVersionComment() { this.inVersionComment = true; } diff --git a/sql/mysql/Oracle/JavaScript/MySQLParserBase.js b/sql/mysql/Oracle/JavaScript/MySQLParserBase.js index 9098a5b5d7..2604643d7e 100644 --- a/sql/mysql/Oracle/JavaScript/MySQLParserBase.js +++ b/sql/mysql/Oracle/JavaScript/MySQLParserBase.js @@ -15,26 +15,32 @@ export default class MySQLParserBase extends Parser { this.serverVersion = 80200; this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } - /** - * Determines if the given SQL mode is currently active in the lexer. - * - * @param mode The mode to check. - * - * @returns True if the mode is one of the currently active modes. - */ - isSqlModeActive(mode) { - return this.sqlModes.has(mode); - } - isPureIdentifier() { - return this.isSqlModeActive(SqlMode.AnsiQuotes); - } - isTextStringLiteral() { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); - } - isStoredRoutineBody() { - return this.serverVersion >= 80032 && this.supportMle; - } - isSelectStatementWithInto() { - return this.serverVersion >= 80024 && this.serverVersion < 80031; - } + isSqlModeActive(mode) { return this.sqlModes.has(mode); } + isPureIdentifier() { return this.isSqlModeActive(SqlMode.AnsiQuotes); } + isTextStringLiteral() { return !this.isSqlModeActive(SqlMode.AnsiQuotes); } + isStoredRoutineBody() { return this.serverVersion >= 80032 && this.supportMle; } + isSelectStatementWithInto() { return this.serverVersion >= 80024 && this.serverVersion < 80031; } + isServerVersionGe80004() { return this.serverVersion >= 80004; } + isServerVersionGe80011() { return this.serverVersion >= 80011; } + isServerVersionGe80013() { return this.serverVersion >= 80013; } + isServerVersionGe80014() { return this.serverVersion >= 80014; } + isServerVersionGe80016() { return this.serverVersion >= 80016; } + isServerVersionGe80017() { return this.serverVersion >= 80017; } + isServerVersionGe80018() { return this.serverVersion >= 80018; } + isServerVersionGe80019() { return this.serverVersion >= 80019; } + isServerVersionGe80024() { return this.serverVersion >= 80024; } + isServerVersionGe80025() { return this.serverVersion >= 80025; } + isServerVersionGe80027() { return this.serverVersion >= 80027; } + isServerVersionGe80031() { return this.serverVersion >= 80031; } + isServerVersionGe80032() { return this.serverVersion >= 80032; } + isServerVersionGe80100() { return this.serverVersion >= 80100; } + isServerVersionGe80200() { return this.serverVersion >= 80200; } + isServerVersionLt80011() { return this.serverVersion < 80011; } + isServerVersionLt80012() { return this.serverVersion < 80012; } + isServerVersionLt80014() { return this.serverVersion < 80014; } + isServerVersionLt80016() { return this.serverVersion < 80016; } + isServerVersionLt80017() { return this.serverVersion < 80017; } + isServerVersionLt80024() { return this.serverVersion < 80024; } + isServerVersionLt80025() { return this.serverVersion < 80025; } + isServerVersionLt80031() { return this.serverVersion < 80031; } } diff --git a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts index 36e07da86a..57638aaaf3 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLLexerBase.ts @@ -19,6 +19,8 @@ export default abstract class MySQLLexerBase extends Lexer { /** Enable Multi Language Extension support. */ public supportMle = true; + public justEmittedDot = false; + public charSets: Set = new Set(); // Used to check repertoires. protected inVersionComment = false; @@ -229,11 +231,25 @@ export default abstract class MySQLLexerBase extends Lexer { * Creates a DOT token in the token stream. */ protected emitDot(): void { + let len = this.text.length; let t = new CommonToken([this, this._input], MySQLLexer.DOT_SYMBOL, 0, this._tokenStartCharIndex, this._tokenStartCharIndex); this.pendingTokens.push(t); - ++this.column; + t.text = "."; + t.column = t.column - len; +// ++this.column; ++this._tokenStartCharIndex; + this.justEmittedDot = true; + } + + public override emit(): Token + { + let t = super.emit(); + if (this.justEmittedDot) { + t.column = t.column + 1; + this.justEmittedDot = false; + } + return t; } public isServerVersionLt80024(): boolean @@ -491,14 +507,6 @@ export default abstract class MySQLLexerBase extends Lexer { return !this.isSqlModeActive(SqlMode.NoBackslashEscapes); } - public override emit(): Token - { - let t = super.emit(); - if (t.type == MySQLLexer.WHITESPACE) - t.channel = Token.HIDDEN_CHANNEL; - return t; - } - public startInVersionComment(): void { this.inVersionComment = true; diff --git a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts index 5744763f15..2f43334f69 100644 --- a/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts +++ b/sql/mysql/Oracle/TypeScript/MySQLParserBase.ts @@ -22,34 +22,32 @@ export default abstract class MySQLParserBase extends Parser { this.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES"); } - /** - * Determines if the given SQL mode is currently active in the lexer. - * - * @param mode The mode to check. - * - * @returns True if the mode is one of the currently active modes. - */ - public isSqlModeActive(mode: SqlMode): boolean { - return this.sqlModes.has(mode); - } - - public isPureIdentifier(): boolean - { - return this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public isTextStringLiteral(): boolean - { - return !this.isSqlModeActive(SqlMode.AnsiQuotes); - } - - public isStoredRoutineBody(): boolean - { - return this.serverVersion >= 80032 && this.supportMle; - } - - public isSelectStatementWithInto(): boolean - { - return this.serverVersion >= 80024 && this.serverVersion < 80031; - } + public isSqlModeActive(mode: SqlMode): boolean { return this.sqlModes.has(mode); } + public isPureIdentifier(): boolean { return this.isSqlModeActive(SqlMode.AnsiQuotes); } + public isTextStringLiteral(): boolean { return !this.isSqlModeActive(SqlMode.AnsiQuotes); } + public isStoredRoutineBody(): boolean { return this.serverVersion >= 80032 && this.supportMle; } + public isSelectStatementWithInto(): boolean { return this.serverVersion >= 80024 && this.serverVersion < 80031; } + public isServerVersionGe80004(): boolean { return this.serverVersion >= 80004; } + public isServerVersionGe80011(): boolean { return this.serverVersion >= 80011; } + public isServerVersionGe80013(): boolean { return this.serverVersion >= 80013; } + public isServerVersionGe80014(): boolean { return this.serverVersion >= 80014; } + public isServerVersionGe80016(): boolean { return this.serverVersion >= 80016; } + public isServerVersionGe80017(): boolean { return this.serverVersion >= 80017; } + public isServerVersionGe80018(): boolean { return this.serverVersion >= 80018; } + public isServerVersionGe80019(): boolean { return this.serverVersion >= 80019; } + public isServerVersionGe80024(): boolean { return this.serverVersion >= 80024; } + public isServerVersionGe80025(): boolean { return this.serverVersion >= 80025; } + public isServerVersionGe80027(): boolean { return this.serverVersion >= 80027; } + public isServerVersionGe80031(): boolean { return this.serverVersion >= 80031; } + public isServerVersionGe80032(): boolean { return this.serverVersion >= 80032; } + public isServerVersionGe80100(): boolean { return this.serverVersion >= 80100; } + public isServerVersionGe80200(): boolean { return this.serverVersion >= 80200; } + public isServerVersionLt80011(): boolean { return this.serverVersion < 80011; } + public isServerVersionLt80012(): boolean { return this.serverVersion < 80012; } + public isServerVersionLt80014(): boolean { return this.serverVersion < 80014; } + public isServerVersionLt80016(): boolean { return this.serverVersion < 80016; } + public isServerVersionLt80017(): boolean { return this.serverVersion < 80017; } + public isServerVersionLt80024(): boolean { return this.serverVersion < 80024; } + public isServerVersionLt80025(): boolean { return this.serverVersion < 80025; } + public isServerVersionLt80031(): boolean { return this.serverVersion < 80031; } } From 3df956db393fdb33b6b1f34d190f3696c38504c1 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 21:54:29 -0500 Subject: [PATCH 27/29] Fixed Python3 port. --- sql/mysql/Oracle/Python3/MySQLLexerBase.py | 28 ++++----- sql/mysql/Oracle/Python3/MySQLParserBase.py | 68 +++++++++++++++++++++ sql/mysql/Oracle/desc.xml | 2 +- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/sql/mysql/Oracle/Python3/MySQLLexerBase.py b/sql/mysql/Oracle/Python3/MySQLLexerBase.py index 5e9a2796b9..5aae2ad7eb 100644 --- a/sql/mysql/Oracle/Python3/MySQLLexerBase.py +++ b/sql/mysql/Oracle/Python3/MySQLLexerBase.py @@ -150,6 +150,18 @@ def emitDot(self): self._tokenStartColumn += 1 self.justEmitedDot = True + def emit(self): + t = super().emit() + if self.justEmitedDot: + text = t.text + text = text[1:] + t.text = text + t.start = t.start + 1 + # print(f"hi {text}", file=sys.stderr) + #print(f"hi {t.text}", file=sys.stderr) + self.justEmitedDot = False + return t + def isServerVersionLt80024(self): return self.serverVersion < 80024 @@ -307,22 +319,6 @@ def isDoubleQuotedText(self): def isSingleQuotedText(self): return not self.isSqlModeActive(SqlMode.NoBackslashEscapes) - def emit(self): - if self._text is None: - text = self._text - elif self._text is NoneType: - text = self._text - elif self.justEmitedDot: - text = self._text[1:] - else: - text = self._text - token = self._factory.create(self._tokenFactorySourcePair, self._type, self._text, self._channel, self._tokenStartCharIndex, - self.getCharIndex()-1, self._tokenStartLine, self._tokenStartColumn) - self.emitToken(token) - self.justEmitedDot = False - self.emitToken(token) - return token - def startInVersionComment(self): self.inVersionComment = True diff --git a/sql/mysql/Oracle/Python3/MySQLParserBase.py b/sql/mysql/Oracle/Python3/MySQLParserBase.py index b6051d4d87..8dde1f8c39 100644 --- a/sql/mysql/Oracle/Python3/MySQLParserBase.py +++ b/sql/mysql/Oracle/Python3/MySQLParserBase.py @@ -62,3 +62,71 @@ def sqlModeFromString(self, modes): elif mode in {"HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40"}: self.sqlModes.add(SqlMode.HighNotPrecedence) + def isServerVersionGe80004(self): + return self.serverVersion >= 80004 + + def isServerVersionGe80011(self): + return self.serverVersion >= 80011 + + def isServerVersionGe80013(self): + return self.serverVersion >= 80013 + + def isServerVersionGe80014(self): + return self.serverVersion >= 80014 + + def isServerVersionGe80016(self): + return self.serverVersion >= 80016 + + def isServerVersionGe80017(self): + return self.serverVersion >= 80017 + + def isServerVersionGe80018(self): + return self.serverVersion >= 80018 + + def isServerVersionGe80019(self): + return self.serverVersion >= 80019 + + def isServerVersionGe80024(self): + return self.serverVersion >= 80024 + + def isServerVersionGe80025(self): + return self.serverVersion >= 80025 + + def isServerVersionGe80027(self): + return self.serverVersion >= 80027 + + def isServerVersionGe80031(self): + return self.serverVersion >= 80031 + + def isServerVersionGe80032(self): + return self.serverVersion >= 80032 + + def isServerVersionGe80100(self): + return self.serverVersion >= 80100 + + def isServerVersionGe80200(self): + return self.serverVersion >= 80200 + + def isServerVersionLt80011(self): + return self.serverVersion < 80011 + + def isServerVersionLt80012(self): + return self.serverVersion < 80012 + + def isServerVersionLt80014(self): + return self.serverVersion < 80014 + + def isServerVersionLt80016(self): + return self.serverVersion < 80016 + + def isServerVersionLt80017(self): + return self.serverVersion < 80017 + + def isServerVersionLt80024(self): + return self.serverVersion < 80024 + + def isServerVersionLt80025(self): + return self.serverVersion < 80025 + + def isServerVersionLt80031(self): + return self.serverVersion < 80031 diff --git a/sql/mysql/Oracle/desc.xml b/sql/mysql/Oracle/desc.xml index b6341320c0..6665f1e000 100644 --- a/sql/mysql/Oracle/desc.xml +++ b/sql/mysql/Oracle/desc.xml @@ -1,5 +1,5 @@ - Antlr4ng;Cpp;CSharp;Dart;Java;JavaScript;Python3;TypeScript + Antlr4ng;Cpp;CSharp;Dart;Go;Java;JavaScript;Python3;TypeScript examples/**/*.sql From 50d967d2835ab94eb49f41c27e85e6fb150c124e Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Mon, 2 Dec 2024 22:37:00 -0500 Subject: [PATCH 28/29] Refactor Python3 SqlModes out. --- sql/mysql/Oracle/Python3/MySQLLexerBase.py | 21 ++---------------- sql/mysql/Oracle/Python3/MySQLParserBase.py | 21 ++---------------- sql/mysql/Oracle/Python3/SqlModes.py | 24 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 38 deletions(-) create mode 100644 sql/mysql/Oracle/Python3/SqlModes.py diff --git a/sql/mysql/Oracle/Python3/MySQLLexerBase.py b/sql/mysql/Oracle/Python3/MySQLLexerBase.py index 5aae2ad7eb..94a2b30f22 100644 --- a/sql/mysql/Oracle/Python3/MySQLLexerBase.py +++ b/sql/mysql/Oracle/Python3/MySQLLexerBase.py @@ -7,6 +7,7 @@ else: from typing.io import TextIO from SqlMode import SqlMode +from SqlModes import SqlModes from MySQLParser import MySQLParser class MySQLLexerBase(Lexer): @@ -23,35 +24,17 @@ class MySQLLexerBase(Lexer): def __init__(self, input=None, output:TextIO = sys.stdout): super().__init__(input, output) self.serverVersion = 0 - self.sqlModes = set() + self.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES") self.supportMle = True self.charSets = set() self.inVersionComment = False self.pendingTokens = deque() self.justEmitedDot = False self.serverVersion = 80200; - self.sqlModeFromString("ANSI_QUOTES"); def isSqlModeActive(self, mode): return mode in self.sqlModes - def sqlModeFromString(self, modes): - self.sqlModes.clear() - parts = modes.upper().split(",") - for mode in parts: - if mode in {"ANSI", "DB2", "MAXDB", "MSSQL", "ORACLE", "POSTGRESQL"}: - self.sqlModes.update({SqlMode.AnsiQuotes, SqlMode.PipesAsConcat, SqlMode.IgnoreSpace}) - elif mode == "ANSI_QUOTES": - self.sqlModes.add(SqlMode.AnsiQuotes) - elif mode == "PIPES_AS_CONCAT": - self.sqlModes.add(SqlMode.PipesAsConcat) - elif mode == "NO_BACKSLASH_ESCAPES": - self.sqlModes.add(SqlMode.NoBackslashEscapes) - elif mode == "IGNORE_SPACE": - self.sqlModes.add(SqlMode.IgnoreSpace) - elif mode in {"HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40"}: - self.sqlModes.add(SqlMode.HighNotPrecedence) - def reset(self): self.inVersionComment = False super().reset() diff --git a/sql/mysql/Oracle/Python3/MySQLParserBase.py b/sql/mysql/Oracle/Python3/MySQLParserBase.py index 8dde1f8c39..d0d3cda04a 100644 --- a/sql/mysql/Oracle/Python3/MySQLParserBase.py +++ b/sql/mysql/Oracle/Python3/MySQLParserBase.py @@ -6,15 +6,15 @@ else: from typing.io import TextIO from SqlMode import SqlMode +from SqlModes import SqlModes class MySQLParserBase(Parser): def __init__(self, input:TokenStream, output:TextIO = sys.stdout): super().__init__(input, output) self.serverVersion = 0 - self.sqlModes = set() + self.sqlModes = SqlModes.sqlModeFromString("ANSI_QUOTES") self.supportMle = True self.serverVersion = 80200; - self.sqlModeFromString("ANSI_QUOTES"); def isSqlModeActive(self, mode): """ @@ -45,23 +45,6 @@ def isStoredRoutineBody(self): def isSelectStatementWithInto(self): return self.serverVersion >= 80024 and self.serverVersion < 80031; - def sqlModeFromString(self, modes): - self.sqlModes.clear() - parts = modes.upper().split(",") - for mode in parts: - if mode in {"ANSI", "DB2", "MAXDB", "MSSQL", "ORACLE", "POSTGRESQL"}: - self.sqlModes.update({SqlMode.AnsiQuotes, SqlMode.PipesAsConcat, SqlMode.IgnoreSpace}) - elif mode == "ANSI_QUOTES": - self.sqlModes.add(SqlMode.AnsiQuotes) - elif mode == "PIPES_AS_CONCAT": - self.sqlModes.add(SqlMode.PipesAsConcat) - elif mode == "NO_BACKSLASH_ESCAPES": - self.sqlModes.add(SqlMode.NoBackslashEscapes) - elif mode == "IGNORE_SPACE": - self.sqlModes.add(SqlMode.IgnoreSpace) - elif mode in {"HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40"}: - self.sqlModes.add(SqlMode.HighNotPrecedence) - def isServerVersionGe80004(self): return self.serverVersion >= 80004 diff --git a/sql/mysql/Oracle/Python3/SqlModes.py b/sql/mysql/Oracle/Python3/SqlModes.py new file mode 100644 index 0000000000..350b91bee7 --- /dev/null +++ b/sql/mysql/Oracle/Python3/SqlModes.py @@ -0,0 +1,24 @@ +import sys +from readchar import readchar +from SqlMode import SqlMode + +class SqlModes: + + @staticmethod + def sqlModeFromString(modes): + sqlModes = set() + parts = modes.upper().split(",") + for mode in parts: + if mode in {"ANSI", "DB2", "MAXDB", "MSSQL", "ORACLE", "POSTGRESQL"}: + sqlModes.update({SqlMode.AnsiQuotes, SqlMode.PipesAsConcat, SqlMode.IgnoreSpace}) + elif mode == "ANSI_QUOTES": + sqlModes.add(SqlMode.AnsiQuotes) + elif mode == "PIPES_AS_CONCAT": + sqlModes.add(SqlMode.PipesAsConcat) + elif mode == "NO_BACKSLASH_ESCAPES": + sqlModes.add(SqlMode.NoBackslashEscapes) + elif mode == "IGNORE_SPACE": + sqlModes.add(SqlMode.IgnoreSpace) + elif mode in {"HIGH_NOT_PRECEDENCE", "MYSQL323", "MYSQL40"}: + sqlModes.add(SqlMode.HighNotPrecedence) + return sqlModes From 2f0592552b7df419267d5d4e646f24d0523ae5c5 Mon Sep 17 00:00:00 2001 From: Ken Domino Date: Tue, 3 Dec 2024 21:43:19 -0500 Subject: [PATCH 29/29] Update readme. --- sql/mysql/Oracle/README.md | 54 ++++------ sql/mysql/Oracle/times.svg | 200 +++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 37 deletions(-) create mode 100644 sql/mysql/Oracle/times.svg diff --git a/sql/mysql/Oracle/README.md b/sql/mysql/Oracle/README.md index 7d3948d9a0..12f3cf25a6 100644 --- a/sql/mysql/Oracle/README.md +++ b/sql/mysql/Oracle/README.md @@ -2,49 +2,29 @@ ## General -For more than a decade, the MySQL GUI development tools team at Oracle has provided the open source -[MySQL Workbench](https://github.com/mysql/mysql-workbench), which uses ANTLR4 for all MySQL code parsing tasks. This requires to translate all changes from the -[MySQL server grammar](https://github.com/mysql/mysql-server/blob/8.0/sql/sql_yacc.yy) to ANTLR4, which is an ongoing effort to always stay up-to-date with the latest and greatest server features (like the Multi Language Extension (MLE) for stored routines). The current grammar supports all MySQL versions starting with 8.0. +This parser grammar is derived from the official Oracle grammar posted here in original/, +which is derived from sources in the MySQL Shell for VS Code extension. +https://github.com/mysql/mysql-shell-plugins/tree/8928ada7d9e37a4075291880738983752b315fee/gui/frontend/src/parsing/mysql -Meanwhile, development focus has been shifted to the [MySQL Shell for VS Code extension](https://marketplace.visualstudio.com/items?itemName=Oracle.mysql-shell-for-vs-code), which is the original source of the grammar you can find here. +This grammar is set to recognize version "8.0.200". -Parser generated from this grammar are very fast (given the high ambiguity of the MySQL language). For details see the [ANTLR4 runtime benchmarks](https://github.com/mike-lischke/antlr4-runtime-benchmarks/tree/main/src/mysql) repository. +## License Like all of Oracle's open source, this grammar is released under the GPLv2. -## Correct and Flexible Parsing +## Target Agnostic -In order for applications like MySQL Shell for VS Code to parse MySQL code correctly, some conditions must be considered, namely the currently used MySQL **server version** and the active **SQL modes** (e.g. to distinguish between identifiers and double-quoted strings, depending on the ANSI mode setting). There are also some peculiarities to consider: +This grammar is "target agnostic." Unaltered, the .g4 files will not work for +Antlr4ng, Cpp, Go, and Python3. You will need to first run `python transformGrammar.py` +provided in the target-specific directory. The script modifies the .g4 files +for the port. -- [String literal concatenation and Character set introducers (aka underscore char sets or string repertoires)](https://dev.mysql.com/doc/refman/8.0/en/string-literals.html). -- [Keyword after dot](https://dev.mysql.com/doc/refman/8.0/en/keywords.html) -- [Built-in function name parsing](https://dev.mysql.com/doc/refman/8.0/en/function-resolution.html). The IGNORE_SPACE SQL mode is properly handled. -- [Version Comments](https://dev.mysql.com/doc/refman/8.0/en/comments.html), like `CREATE TABLE t1(a INT, KEY (a)) /*!50110 KEY_BLOCK_SIZE=1024 */;` +## Modifying this grammar +This grammar is current hand-written. The plan is to generate the ports directly +from the sources at https://github.com/mysql/mysql-shell-plugins. -The server version and SQL mode can be toggled at runtime, allowing the use of a single parser with different version/mode settings and providing better error messages (like for [a feature that is only valid for a specific version](https://github.com/mysql/mysql-shell-plugins/blob/master/gui/frontend/src/parsing/mysql/MySQLErrorListener.ts#L109)). +## Issues +* The grammar is ambiguous, but generally performs well, except for bitrix_queries_cut.sql, which contains ~3000 ambiguities. -String repertoires require a list of character set identifiers, which must be provided by your implementation. You can get a list of available character sets by running `show character set`. - -## Using the Grammar - -To provide the full feature set the MySQL grammar needs some support code, which is implemented in base classes for both, the MySQL Parser (named `MySQLBaseRecognizer`) and the MySQL Lexer (named `MySQLBaseLexer`). You can find a TypeScript implementation of both classes in the TypeScript/ folder, which should be easy to port over to different runtime languages. - -This folder also contains a demo script that shows how to set up the MySQL lexer and parser and parse some input. It needs the TS runtime antlr4ng (and some additional modules to allow running the demo). For this run the node module installation in the TypeScript/ folder: - -```bash -npm i -``` - -After that you can generate the (TypeScript) parser and lexer files by running: - -```bash -npm run generate -``` - -A new folder is created name `generated`, which contains the new files. Now the demo is ready for execution: - -```bash -npm run demo -``` - -It will run a simple MySQL query and prints its parse tree. +## Performance + diff --git a/sql/mysql/Oracle/times.svg b/sql/mysql/Oracle/times.svg new file mode 100644 index 0000000000..7c2cced37c --- /dev/null +++ b/sql/mysql/Oracle/times.svg @@ -0,0 +1,200 @@ + + + Figure 1 + +Creator: GL2PS 1.4.2, (C) 1999-2020 C. Geuzaine +For: Octave +CreationDate: Tue Dec 03 10:09:56 2024 + + + + + + + + + + + + + + + + + + + + + 1 + + + 2 + + + 3 + + + 4 + + + 5 + + + 6 + + + 7 + + + 8 + + + 9 + + + + + + + + + 0 + + + 10 + + + 20 + + + 30 + + + 40 + + + 50 + + + + + Port + + + Runtime (s) + + + Runtimes for Ports of MySQLParser + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Antlr4ng + + + + + CSharp + + + + + Cpp + + + + + Dart + + + + + Go + + + + + Java + + + + + JavaScript + + + + + TypeScript + + + + + Python3 + + + +