diff --git a/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/access.ts b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/access.ts index 513c62b45b..4c394dab9e 100644 --- a/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/access.ts +++ b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/access.ts @@ -1,11 +1,12 @@ import { XmlBasedJson, XmlParseError } from '../../common/input-format' import { getTokenType, retrieveMetaStructure } from '../../common/meta' -import { RType, RNode, RArgument, RawRType, RFunctionCall } from '../../../../model' +import { RType, RNode, RawRType, RFunctionCall } from '../../../../model' import { guard } from '../../../../../../../util/assert' import { splitArrayOn } from '../../../../../../../util/arrays' import { NormalizeConfiguration } from '../data' import { normalizeSingleToken } from './single-element' import { tryToNormalizeArgument } from './functions/argument' +import {InternalScope} from './internal' /** * Normalize the given data as access (e.g., indexing). @@ -53,8 +54,7 @@ export function normalizeAccess(configuration: NormalizeConfiguration, tokens: r flavor: 'named', functionName: { type: RType.Symbol, - // TODO: just lock "internal" here - namespace: undefined, + namespace: InternalScope, lexeme: content, info: {}, content, @@ -70,7 +70,9 @@ export function normalizeAccess(configuration: NormalizeConfiguration, tokens: r // otherwise we have to process const remaining: readonly XmlBasedJson[] = tokens.slice(2, tokens.length - closingLength) - const splitAccessOnComma = splitArrayOn(remaining, elem => elem.name === RawRType.Comma) + const splitAccessOnComma = splitArrayOn(remaining, elem => + getTokenType(configuration.tokenMap, elem) === RawRType.Comma + ) const parsedAccess: (RNode | undefined)[] = splitAccessOnComma.map((elems: readonly XmlBasedJson[]) => elems.length === 0 ? undefined : normalizeAccessArgument(configuration, elems) @@ -84,8 +86,7 @@ export function normalizeAccess(configuration: NormalizeConfiguration, tokens: r flavor: 'named', functionName: { type: RType.Symbol, - // TODO: just lock "internal" here - namespace: undefined, + namespace: InternalScope, lexeme: content, info: {}, content, @@ -100,7 +101,7 @@ export function normalizeAccess(configuration: NormalizeConfiguration, tokens: r } -function normalizeAccessArgument(config: NormalizeConfiguration, elements: readonly XmlBasedJson[]): RArgument { +function normalizeAccessArgument(config: NormalizeConfiguration, elements: readonly XmlBasedJson[]): RNode { const res = tryToNormalizeArgument(config, elements) guard(res !== undefined, () => `expected one access result in access as argument, yet received ${JSON.stringify(res)} for ${JSON.stringify(elements)}`) return res diff --git a/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/functions/argument.ts b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/functions/argument.ts index 1ad2ded634..8cd4706ec4 100644 --- a/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/functions/argument.ts +++ b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/functions/argument.ts @@ -1,6 +1,6 @@ import { XmlBasedJson, XmlParseError } from '../../../common/input-format' import { getTokenType, retrieveMetaStructure } from '../../../common/meta' -import { RNode, RType, RSymbol, RArgument, RawRType } from '../../../../../model' +import { RNode, RType, RSymbol, RawRType } from '../../../../../model' import { NormalizeConfiguration } from '../../data' import { log } from '../../../../../../../../util/log' import { guard } from '../../../../../../../../util/assert' @@ -16,7 +16,7 @@ import { normalizeSingleToken } from '../single-element' * * @returns The parsed argument or `undefined` if the given object is not an argument. */ -export function tryToNormalizeArgument(configuration: NormalizeConfiguration, objs: readonly XmlBasedJson[]): RArgument | undefined { +export function tryToNormalizeArgument(configuration: NormalizeConfiguration, objs: readonly XmlBasedJson[]): RNode | undefined { if(objs.length < 1 || objs.length > 3) { log.warn(`Either [expr|value], [SYMBOL_SUB, EQ_SUB], or [SYMBOL_SUB, EQ_SUB, expr], but got: ${objs.map(o => JSON.stringify(o)).join(', ')}`) return undefined @@ -51,16 +51,20 @@ export function tryToNormalizeArgument(configuration: NormalizeConfiguration, ob guard(parsedValue !== undefined && parsedValue?.type !== RType.Delimiter, () => `[argument] parsed value must not be undefined, yet: ${JSON.stringify(objs)}`) - return { - type: RType.Argument, - location, - lexeme: content, - name, - value: parsedValue ?? undefined, - info: { - fullRange: location, - fullLexeme: content, - additionalTokens: [] + if(name === undefined) { + return parsedValue ?? undefined + } else { + return { + type: RType.Argument, + location, + lexeme: content, + name, + value: parsedValue ?? undefined, + info: { + fullRange: location, + fullLexeme: content, + additionalTokens: [] + } } } } diff --git a/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/internal.ts b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/internal.ts new file mode 100644 index 0000000000..5709d7ca0e --- /dev/null +++ b/src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/internal.ts @@ -0,0 +1 @@ +export const InternalScope = '.$internal' diff --git a/test/functionality/r-bridge/lang/ast/parse-access.ts b/test/functionality/r-bridge/lang/ast/parse-access.ts index 7829d245d1..242e2d3475 100644 --- a/test/functionality/r-bridge/lang/ast/parse-access.ts +++ b/test/functionality/r-bridge/lang/ast/parse-access.ts @@ -3,6 +3,7 @@ import { exprList, numVal } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { DESUGAR_NORMALIZE, NORMALIZE } from '../../../../../src/core/steps/all/core/10-normalize' +import {InternalScope} from '../../../../../src/r-bridge/lang-4.x/ast/parser/xml/v2/internal/internal' describe('Parse value access', withShell(shell => { describe('Single bracket', () => { @@ -37,7 +38,7 @@ describe('Parse value access', withShell(shell => { content: '[', info: {}, location: rangeFrom(1, 2, 1, 2), - namespace: undefined + namespace: InternalScope }, location: rangeFrom(1, 2, 1, 2), flavor: 'named', @@ -53,35 +54,72 @@ describe('Parse value access', withShell(shell => { }) } ]) - assertAst('One value', shell, 'a[1]', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 2), - lexeme: '[', - operator: '[', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'a', - content: 'a', - info: {} + assertAst('One value', shell, 'a[1]', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 2), + lexeme: '[', + operator: '[', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'a', + content: 'a', + info: {} + }, + access: [{ + type: RType.Argument, + location: rangeFrom(1, 3, 1, 3), + lexeme: '1', + name: undefined, + info: {}, + value: { + type: RType.Number, + location: rangeFrom(1, 3, 1, 3), + lexeme: '1', + content: numVal(1), + info: {} + } + }] + }) }, - access: [{ - type: RType.Argument, - location: rangeFrom(1, 3, 1, 3), - lexeme: '1', - name: undefined, - info: {}, - value: { - type: RType.Number, - location: rangeFrom(1, 3, 1, 3), - lexeme: '1', - content: numVal(1), - info: {} - } - }] - })) + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + info: {}, + lexeme: '[', + functionName: { + type: RType.Symbol, + lexeme: '[', + content: '[', + info: {}, + location: rangeFrom(1, 2, 1, 2), + namespace: InternalScope + }, + location: rangeFrom(1, 2, 1, 2), + flavor: 'named', + arguments: [{ + type: RType.Symbol, + lexeme: 'a', + content: 'a', + info: {}, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined + }, { + type: RType.Number, + location: rangeFrom(1, 3, 1, 3), + lexeme: '1', + content: numVal(1), + info: {} + }] + }) + } + ]) assertAst('One variable', shell, 'a[x]', exprList({ type: RType.Access, location: rangeFrom(1, 2, 1, 2), @@ -342,157 +380,380 @@ describe('Parse value access', withShell(shell => { })) }) describe('Double bracket', () => { - assertAst('Empty', shell, 'b[[]]', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 3), - lexeme: '[[', - operator: '[[', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'b', - content: 'b', - info: {} - }, - access: [] - })) - assertAst('One element', shell, 'b[[5]]', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 3), - lexeme: '[[', - operator: '[[', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'b', - content: 'b', - info: {} + assertAst('Empty', shell, 'b[[]]', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + operator: '[[', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, + access: [] + }) }, - access: [{ - type: RType.Argument, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - name: undefined, - info: {}, - value: { - type: RType.Number, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - content: numVal(5), - info: {} - } - }] - })) - assertAst('Multiple', shell, 'b[[5,3]]', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 3), - lexeme: '[[', - operator: '[[', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'b', - content: 'b', - info: {} + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + info: {}, + lexeme: '[[', + functionName: { + type: RType.Symbol, + lexeme: '[[', + content: '[[', + info: {}, + location: rangeFrom(1, 2, 1, 3), + namespace: InternalScope + }, + location: rangeFrom(1, 2, 1, 3), + flavor: 'named', + arguments: [{ + type: RType.Symbol, + lexeme: 'b', + content: 'b', + info: {}, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined + }] + }) + } + ]) + assertAst('One element', shell, 'b[[5]]', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + operator: '[[', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, + access: [{ + type: RType.Argument, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + name: undefined, + info: {}, + value: { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + } + }] + }) }, - access: [{ - type: RType.Argument, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - name: undefined, - info: {}, - value: { - type: RType.Number, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - content: numVal(5), - info: {} - } - }, { - type: RType.Argument, - location: rangeFrom(1, 6, 1, 6), - lexeme: '3', - name: undefined, - info: {}, - value: { - type: RType.Number, - location: rangeFrom(1, 6, 1, 6), - lexeme: '3', - content: numVal(3), - info: {} - } - }] - })) - assertAst('Multiple with empty', shell, 'b[[5,,]]', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 3), - lexeme: '[[', - operator: '[[', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'b', - content: 'b', - info: {} + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + info: {}, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + functionName: { + type: RType.Symbol, + lexeme: '[[', + content: '[[', + info: {}, + location: rangeFrom(1, 2, 1, 3), + namespace: InternalScope + }, + flavor: 'named', + arguments: [{ + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + }] + }) + } + ]) + assertAst('Multiple', shell, 'b[[5,3]]', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + operator: '[[', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, + access: [{ + type: RType.Argument, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + name: undefined, + info: {}, + value: { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + } + }, { + type: RType.Argument, + location: rangeFrom(1, 6, 1, 6), + lexeme: '3', + name: undefined, + info: {}, + value: { + type: RType.Number, + location: rangeFrom(1, 6, 1, 6), + lexeme: '3', + content: numVal(3), + info: {} + } + }] + }) }, - access: [{ + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + info: {}, + functionName: { + type: RType.Symbol, + lexeme: '[[', + content: '[[', + info: {}, + location: rangeFrom(1, 2, 1, 3), + namespace: InternalScope + }, + flavor: 'named', + arguments: [{ + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + }, { + type: RType.Number, + location: rangeFrom(1, 6, 1, 6), + lexeme: '3', + content: numVal(3), + info: {} + }] + }) + } + ]) + assertAst('Multiple with empty', shell, 'b[[5,,]]', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + operator: '[[', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, + access: [{ - type: RType.Argument, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - name: undefined, - info: {}, - value: { - type: RType.Number, - location: rangeFrom(1, 4, 1, 4), - lexeme: '5', - content: numVal(5), - info: {} - } - },null,null] - })) + type: RType.Argument, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + name: undefined, + info: {}, + value: { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + } + }, null, null] + }) + }, + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + location: rangeFrom(1, 2, 1, 3), + lexeme: '[[', + info: {}, + functionName: { + type: RType.Symbol, + lexeme: '[[', + content: '[[', + info: {}, + location: rangeFrom(1, 2, 1, 3), + namespace: InternalScope + }, + flavor: 'named', + arguments: [{ + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'b', + content: 'b', + info: {} + }, { + type: RType.Number, + location: rangeFrom(1, 4, 1, 4), + lexeme: '5', + content: numVal(5), + info: {} + }, undefined, undefined] + }) + } + ]) }) describe('Dollar and Slot', () => { - assertAst('Dollar access', shell, 'c$x', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 2), - lexeme: '$', - operator: '$', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'c', - content: 'c', - info: {} + assertAst('Dollar access', shell, 'c$x', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 2), + lexeme: '$', + operator: '$', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'c', + content: 'c', + info: {} + }, + access: 'x' + }) }, - access: 'x' - })) - assertAst('Slot based access', shell, 'd@y', exprList({ - type: RType.Access, - location: rangeFrom(1, 2, 1, 2), - lexeme: '@', - operator: '@', - info: {}, - accessed: { - type: RType.Symbol, - location: rangeFrom(1, 1, 1, 1), - namespace: undefined, - lexeme: 'd', - content: 'd', - info: {} + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + location: rangeFrom(1, 2, 1, 2), + lexeme: '$', + info: {}, + functionName: { + type: RType.Symbol, + lexeme: '$', + content: '$', + info: {}, + location: rangeFrom(1, 2, 1, 2), + namespace: InternalScope + }, + flavor: 'named', + arguments: [{ + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'c', + content: 'c', + info: {} + }, { + type: RType.Symbol, + location: rangeFrom(1, 3, 1, 3), + namespace: undefined, + lexeme: 'x', + content: 'x', + info: {} + }] + }) + } + ]) + assertAst('Slot based access', shell, 'd@y', [ + { + step: NORMALIZE, + wanted: exprList({ + type: RType.Access, + location: rangeFrom(1, 2, 1, 2), + lexeme: '@', + operator: '@', + info: {}, + accessed: { + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'd', + content: 'd', + info: {} + }, + access: 'y' + }) }, - access: 'y' - })) + { + step: DESUGAR_NORMALIZE, + wanted: exprList({ + type: RType.FunctionCall, + location: rangeFrom(1, 2, 1, 2), + lexeme: '@', + info: {}, + functionName: { + type: RType.Symbol, + lexeme: '@', + content: '@', + info: {}, + location: rangeFrom(1, 2, 1, 2), + namespace: InternalScope + }, + flavor: 'named', + arguments: [{ + type: RType.Symbol, + location: rangeFrom(1, 1, 1, 1), + namespace: undefined, + lexeme: 'd', + content: 'd', + info: {} + }, { + type: RType.Symbol, + location: rangeFrom(1, 3, 1, 3), + namespace: undefined, + lexeme: 'y', + content: 'y', + info: {} + }] + }) + } + ]) }) })) -