From db7014ab44be72be66e7d1e1718a8205eda528a5 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 9 Dec 2024 12:25:45 -0800 Subject: [PATCH] Adds support for macro-aware transcoding from text. --- .../amazon/ion/impl/IonReaderTextSystemX.java | 187 +++++++++++++++++- .../amazon/ion/impl/IonReaderTextUserX.java | 46 +---- .../ion/impl/_Private_IonReaderBuilder.java | 9 +- .../ion/impl/_Private_IonReaderFactory.java | 33 +--- .../ion/impl/macro/EExpressionArgsReader.java | 2 +- .../EncodingDirectiveCompilationTest.java | 171 ++++++++++------ .../ion/impl/IonRawTextReaderTest_1_1.java | 6 +- 7 files changed, 315 insertions(+), 139 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java index efe4221a22..c8e66656aa 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -2,6 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; import static com.amazon.ion.impl._Private_ScalarConversions.getValueTypeName; import com.amazon.ion.Decimal; @@ -17,10 +20,13 @@ import com.amazon.ion.IonTimestamp; import com.amazon.ion.IonType; import com.amazon.ion.IonValue; +import com.amazon.ion.MacroAwareIonReader; +import com.amazon.ion.MacroAwareIonWriter; import com.amazon.ion.SymbolTable; import com.amazon.ion.SymbolToken; import com.amazon.ion.Timestamp; import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.UnsupportedIonVersionException; import com.amazon.ion.impl.IonReaderTextRawTokensX.IonReaderTextTokenException; import com.amazon.ion.impl.IonTokenConstsX.CharacterSequence; import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; @@ -38,15 +44,18 @@ import com.amazon.ion.impl.macro.ReaderAdapter; import com.amazon.ion.impl.macro.ReaderAdapterIonReader; import com.amazon.ion.impl.macro.SystemMacro; +import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; /** * This reader calls the {@link IonReaderTextRawX} for low level events. @@ -60,12 +69,14 @@ */ class IonReaderTextSystemX extends IonReaderTextRawX - implements _Private_ReaderWriter + implements _Private_ReaderWriter, MacroAwareIonReader { private static int UNSIGNED_BYTE_MAX_VALUE = 255; SymbolTable _system_symtab; + SymbolTable _symbols; + // The core MacroEvaluator that this core reader delegates to when evaluating a macro invocation. private final MacroEvaluator macroEvaluator = new MacroEvaluator(); @@ -87,9 +98,13 @@ class IonReaderTextSystemX // Indicates whether the reader is currently evaluating an e-expression. boolean isEvaluatingEExpression = false; + // The writer that will perform a macro-aware transcode, if requested. + private MacroAwareIonWriter macroAwareTranscoder = null; + protected IonReaderTextSystemX(UnifiedInputStreamX iis) { _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amazon-ion/ion-java/issues/19 + _symbols = _system_symtab; init_once(); init(iis, IonType.DATAGRAM); } @@ -720,7 +735,7 @@ public SymbolTable getSymbolTable() SymbolTable symtab = super.getSymbolTable(); if (symtab == null) { - symtab = _system_symtab; + symtab = _symbols; } return symtab; } @@ -1132,7 +1147,7 @@ public SymbolTable pop_passed_symbol_table() * @param symbolTable the symbol table to make active. */ protected void setSymbolTable(SymbolTable symbolTable) { - // System readers don't handle symbol tables. + _symbols = symbolTable; } /** @@ -1189,10 +1204,11 @@ private void readEncodingDirective() { SymbolTable current = getSymbolTable(); if (current.isSystemTable()) { // TODO determine the best way to handle the Ion 1.1 system symbols. - newSymbols.addAll(0, SystemSymbols_1_1.allSymbolTexts()); + List withSystemSymbols = new ArrayList<>(SystemSymbols_1_1.allSymbolTexts()); + withSystemSymbols.addAll(newSymbols); setSymbolTable(new LocalSymbolTable( LocalSymbolTableImports.EMPTY, - newSymbols + withSystemSymbols )); } else { LocalSymbolTable currentLocal = (LocalSymbolTable) current; @@ -1371,6 +1387,167 @@ public boolean hasNext() return has_next_system_value(); } + @Override + public void transcodeAllTo(MacroAwareIonWriter writer) throws IOException { + prepareTranscodeTo(writer); + while (transcodeNext()); + } + + @Override + public void prepareTranscodeTo(@NotNull MacroAwareIonWriter writer) { + macroAwareTranscoder = writer; + } + + @Override + public boolean transcodeNext() throws IOException { + // TODO consider improving the readability of this method and its binary counterpart: https://github.com/amazon-ion/ion-java/issues/1004 + if (macroAwareTranscoder == null) { + throw new IllegalArgumentException("prepareTranscodeTo must be called before transcodeNext."); + } + boolean isSystemValue = false; + while (true) { + if (isEvaluatingEExpression) { + if (evaluateNext()) { + if (isSystemValue) { + continue; + } + return !_eof; + } + } else { + nextRaw(); + } + isSystemValue = false; + if (_value_type != null && getDepth() == 0) { + if (IonType.SYMBOL == getType() && handlePossibleIonVersionMarker()) { + // Which IVM to write is inherent to the writer implementation. + // We don't have a single implementation that writes both formats. + macroAwareTranscoder.startEncodingSegmentWithIonVersionMarker(); + isSystemValue = true; + continue; + } + if (minorVersion > 0 && isPositionedOnEncodingDirective()) { + boolean isEncodingDirectiveFromEExpression = isEvaluatingEExpression; + readEncodingDirective(); + macroAwareTranscoder.startEncodingSegmentWithEncodingDirective( + encodingDirectiveReader.getNewMacros(), + encodingDirectiveReader.isMacroTableAppend(), + encodingDirectiveReader.getNewSymbols(), + encodingDirectiveReader.isSymbolTableAppend(), + isEncodingDirectiveFromEExpression + ); + isSystemValue = true; + continue; + } + } + if (_container_is_e_expression) { + expressionArgsReader.beginEvaluatingMacroInvocation(macroEvaluator); + macroEvaluatorIonReader.transcodeArgumentsTo(macroAwareTranscoder); + isEvaluatingEExpression = true; + continue; + } else if (isEvaluatingEExpression) { + // This is an e-expression that yields user values. Its arguments have already been transcoded. + continue; + } + if (_eof) { + return false; + } + transcodeValueLiteral(); + return !_eof; + } + } + + /** + * @return true if the reader is positioned on an Ion 1.0 symbol table; otherwise, false. Note: the caller must + * ensure this is called only at the top level. + */ + boolean isPositionedOnSymbolTable() { + return _annotation_count > 0 && (ION_SYMBOL_TABLE.equals(_annotations[0].getText()) || ION_SYMBOL_TABLE_SID == _annotations[0].getSid()); + } + + // Matches "$ion_x_y", where x and y are integers. + private static final Pattern ION_VERSION_MARKER_REGEX = Pattern.compile("^\\$ion_[0-9]+_[0-9]+$"); + + /** + * @param text the text of a symbol value. + * @return true if the text denotes an IVM; otherwise, false. + */ + static boolean isIonVersionMarker(String text) + { + return text != null && ION_VERSION_MARKER_REGEX.matcher(text).matches(); + } + + /** + * Resets the symbol table after an IVM is encountered. May be overridden if additional side effects are required. + */ + void symbol_table_reset() { + setSymbolTable(_system_symtab); + } + + /** + * Determines whether the top-level symbol value on which the reader is positioned is an Ion version marker. + * If it is, sets the reader's Ion version accordingly and resets the symbol table. + * @return true if the symbol represented an Ion version marker; otherwise, false. Note: the caller must + * ensure this is called only at the top level with the reader positioned on a symbol value. + */ + boolean handlePossibleIonVersionMarker() { + if (_annotation_count == 0) + { + // $ion_1_0 is read as an IVM only if it is not annotated + String version = symbolValue().getText(); + if (isIonVersionMarker(version)) + { + if (ION_1_0.equals(version) || "$ion_1_1".equals(version)) + { + setMinorVersion(version.charAt(version.length() - 1) - '0'); + if (_value_keyword != IonTokenConstsX.KEYWORD_sid) + { + symbol_table_reset(); + } + _has_next_called = false; + } + else + { + throw new UnsupportedIonVersionException(version); + } + return true; + } + } + return false; + } + + /** + * Transcodes a value literal to the macroAwareTranscoder. The caller must ensure that the reader is positioned + * on a value literal (i.e. a scalar or container value not expanded from an e-expression) before calling this + * method. + * @throws IOException if thrown by the writer during transcoding. + */ + private void transcodeValueLiteral() throws IOException { + if (getDepth() == 0 && isPositionedOnSymbolTable()) { + if (minorVersion > 0) { + // TODO finalize handling of Ion 1.0-style symbol tables in Ion 1.1: https://github.com/amazon-ion/ion-java/issues/1002 + throw new IonException("Macro-aware transcoding of Ion 1.1 data containing Ion 1.0-style symbol tables not yet supported."); + } + // Ion 1.0 symbol tables are transcoded verbatim for now; this may change depending on the resolution to + // https://github.com/amazon-ion/ion-java/issues/1002. + macroAwareTranscoder.writeValue(this); + } else if (IonType.isContainer(getType()) && !isNullValue()) { + // Containers need to be transcoded recursively to avoid expanding macro invocations at any depth. + if (isInStruct()) { + macroAwareTranscoder.setFieldNameSymbol(getFieldNameSymbol()); + } + macroAwareTranscoder.setTypeAnnotationSymbols(getTypeAnnotationSymbols()); + macroAwareTranscoder.stepIn(getType()); + super.stepIn(); + while (transcodeNext()); // TODO make this iterative. + super.stepOut(); + macroAwareTranscoder.stepOut(); + } else { + // The reader is now positioned on a scalar literal. Write the value. + // Note: writeValue will include any field name and/or annotations on the scalar. + macroAwareTranscoder.writeValue(this); + } + } + @Override public void stepIn() { if (isEvaluatingEExpression) { diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java index d0984312f7..23d32370e6 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java @@ -2,9 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl; -import static com.amazon.ion.SystemSymbols.ION_1_0; -import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; import com.amazon.ion.IonCatalog; import com.amazon.ion.IonType; @@ -16,8 +13,6 @@ import com.amazon.ion.SymbolToken; import com.amazon.ion.TextSpan; import com.amazon.ion.UnknownSymbolException; -import com.amazon.ion.UnsupportedIonVersionException; -import java.util.regex.Pattern; /** * The text user reader add support for symbols and recognizes, @@ -44,7 +39,6 @@ class IonReaderTextUserX extends IonReaderTextSystemX implements _Private_ReaderWriter { - private static final Pattern ION_VERSION_MARKER_REGEX = Pattern.compile("^\\$ion_[0-9]+_[0-9]+$"); /** * This is the physical start-of-stream offset when this reader was created. @@ -56,7 +50,6 @@ class IonReaderTextUserX // IonSystem _system; now in IonReaderTextSystemX where it could be null IonCatalog _catalog; - SymbolTable _symbols; protected IonReaderTextUserX(IonCatalog catalog, @@ -77,10 +70,6 @@ protected IonReaderTextUserX(IonCatalog catalog, this(catalog, lstFactory, uis, 0); } - @Override - protected void setSymbolTable(SymbolTable symbolTable) { - _symbols = symbolTable; - } /** * this looks forward to see if there is an upcoming value @@ -124,7 +113,7 @@ private final boolean has_next_user_value() if (_value_type != null && !isNullValue() && IonType.DATAGRAM.equals(getContainerType())) { switch (_value_type) { case STRUCT: - if (_annotation_count > 0 && (ION_SYMBOL_TABLE.equals(_annotations[0].getText()) || ION_SYMBOL_TABLE_SID == _annotations[0].getSid())) { + if (isPositionedOnSymbolTable()) { setSymbolTable(_lstFactory.newLocalSymtab(_catalog, this, true)); @@ -133,28 +122,7 @@ private final boolean has_next_user_value() } break; case SYMBOL: - if (_annotation_count == 0) - { - // $ion_1_0 is read as an IVM only if it is not annotated - String version = symbolValue().getText(); - if (isIonVersionMarker(version)) - { - if (ION_1_0.equals(version) || "$ion_1_1".equals(version)) - { - setMinorVersion(version.charAt(version.length() - 1) - '0'); - if (_value_keyword != IonTokenConstsX.KEYWORD_sid) - { - symbol_table_reset(); - push_symbol_table(_system_symtab); // TODO install the correct system symbol table for the active Ion version. - } - _has_next_called = false; - } - else - { - throw new UnsupportedIonVersionException(version); - } - } - } + handlePossibleIonVersionMarker(); break; default: break; @@ -164,17 +132,13 @@ private final boolean has_next_user_value() return (!_eof); } - static boolean isIonVersionMarker(String text) - { - return text != null && ION_VERSION_MARKER_REGEX.matcher(text).matches(); - } - - private final void symbol_table_reset() + @Override + final void symbol_table_reset() { IonType t = next(); assert( IonType.SYMBOL.equals(t) ); setSymbolTable(_system_symtab); - return; + push_symbol_table(_system_symtab); // TODO install the correct system symbol table for the active Ion version. } private void validateSymbolToken(SymbolToken symbol) { diff --git a/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java b/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java index 3ee99ff4b4..0868421811 100644 --- a/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java +++ b/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java @@ -21,6 +21,7 @@ import static com.amazon.ion.impl.LocalSymbolTable.DEFAULT_LST_FACTORY; import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; import static com.amazon.ion.impl._Private_IonReaderFactory.makeReaderText; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeSystemReaderText; /** * {@link IonReaderBuilder} extension for internal use only. @@ -364,9 +365,7 @@ public MacroAwareIonReader buildMacroAware(byte[] ionData) { 0, ionData.length, (builder, data, offset, length) -> new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), data, offset,length), - (catalog, data, offset, length, factory) -> { - throw new UnsupportedOperationException("MacroAwareIonReader is not yet implemented for text data."); - } + (catalog, data, offset, length, factory) -> (IonReaderTextSystemX) makeSystemReaderText(catalog, data, offset, length, factory) ); } @@ -380,9 +379,7 @@ public MacroAwareIonReader buildMacroAware(InputStream ionData) { this, ionData, (builder, source, alreadyRead, alreadyReadOff, alreadyReadLen) -> new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), source, alreadyRead, alreadyReadOff, alreadyReadLen), - (catalog, source, factory) -> { - throw new UnsupportedOperationException("MacroAwareIonReader is not yet implemented for text data."); - } + (catalog, source, factory) -> (IonReaderTextSystemX) makeSystemReaderText(catalog, source, factory) ); } } diff --git a/src/main/java/com/amazon/ion/impl/_Private_IonReaderFactory.java b/src/main/java/com/amazon/ion/impl/_Private_IonReaderFactory.java index 1f0e83ce57..04682cc9ee 100644 --- a/src/main/java/com/amazon/ion/impl/_Private_IonReaderFactory.java +++ b/src/main/java/com/amazon/ion/impl/_Private_IonReaderFactory.java @@ -1,18 +1,5 @@ -/* - * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl; import static com.amazon.ion.impl.UnifiedInputStreamX.makeStream; @@ -113,9 +100,9 @@ public static IonReader makeSystemReaderText(InputStream is) ); } - private static IonReader makeSystemReaderText(IonCatalog catalog, - InputStream is, - _Private_LocalSymbolTableFactory lstFactory) + public static IonReader makeSystemReaderText(IonCatalog catalog, + InputStream is, + _Private_LocalSymbolTableFactory lstFactory) { UnifiedInputStreamX uis; try @@ -129,11 +116,11 @@ private static IonReader makeSystemReaderText(IonCatalog catalog, return new IonReaderTextSystemX(uis); } - private static IonReader makeSystemReaderText(IonCatalog catalog, - byte[] bytes, - int offset, - int length, - _Private_LocalSymbolTableFactory lstFactory) { + public static IonReader makeSystemReaderText(IonCatalog catalog, + byte[] bytes, + int offset, + int length, + _Private_LocalSymbolTableFactory lstFactory) { UnifiedInputStreamX uis; try { diff --git a/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java b/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java index a5ac5b10b9..0549b0ef98 100644 --- a/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java +++ b/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java @@ -237,7 +237,7 @@ protected void readValueAsExpression(boolean isImplicitRest, List annotations = getAnnotations(); - if (isImplicitRest) { + if (isImplicitRest && !isContainerAnExpressionGroup()) { readStreamAsExpressionGroup(expressions); } else if (IonType.isContainer(type)) { readContainerValueAsExpression(type, annotations, expressions); diff --git a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java index 1abe539678..27a41e4ea6 100644 --- a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java +++ b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java @@ -331,6 +331,21 @@ public static Arguments[] allCombinations() { return combinations; } + public static Arguments[] allInputFormatsInputTypesAndOutputFormats() { + InputType[] inputTypes = InputType.values(); + StreamType[] streamTypes = StreamType.values(); + Arguments[] combinations = new Arguments[inputTypes.length * streamTypes.length * streamTypes.length]; + int i = 0; + for (InputType inputType : inputTypes) { + for (StreamType inputFormat : streamTypes) { + for (StreamType outputFormat : streamTypes) { + combinations[i++] = Arguments.of(inputType, inputFormat, outputFormat); + } + } + } + return combinations; + } + private static int getSymbolId(Map symbols, String value) { Integer sid = symbols.get(value); return sid == null ? -1 : sid; @@ -675,11 +690,14 @@ private ByteArrayOutputStream macroAwareTranscodeValueByValue( return out; } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void nestedInvocationMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationWithinStruct(StreamType.BINARY, new TreeMap<>()); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void nestedInvocationMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationWithinStruct(inputFormat, new TreeMap<>()); ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 1, false); @@ -696,12 +714,15 @@ public void nestedInvocationMacroAwareTranscode(InputType inputType, StreamType ); } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void multipleNestedInvocationMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void multipleNestedInvocationMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { ByteArrayOutputStream source = new ByteArrayOutputStream(); - IonRawWriter_1_1 writer = StreamType.BINARY.newWriter(source); + IonRawWriter_1_1 writer = inputFormat.newWriter(source); writer.writeIVM(); writeSymbolTableEExpression(false, writer, "foo", "bar", "baz", "zar"); @@ -800,11 +821,14 @@ public void zeroArgMacroThatExpandsToEncodingDirective(InputType inputType, Stre } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void zeroArgMacroThatExpandsToEncodingDirectiveMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = zeroArgMacroThatExpandsToEncodingDirective(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void zeroArgMacroThatExpandsToEncodingDirectiveMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = zeroArgMacroThatExpandsToEncodingDirective(inputFormat); ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 2, true); verifyStream("def abc".getBytes(StandardCharsets.UTF_8), out, outputFormat, @@ -1179,11 +1203,14 @@ private static void verifyMacroAwareTranscode( verifyStream(data, out, streamType, expectations); } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void macroInvocationsNestedWithinParameterMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationsNestedWithinParameter(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void macroInvocationsNestedWithinParameterMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationsNestedWithinParameter(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), @@ -1348,11 +1375,14 @@ public void macroInvocationInMacroDefinition(InputType inputType, StreamType str } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void macroInvocationInMacroDefinitionMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationInMacroDefinition(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void macroInvocationInMacroDefinitionMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationInMacroDefinition(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), @@ -1558,11 +1588,14 @@ public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTable(Input } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationsProduceEncodingDirectivesThatModifySymbolTable(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTableMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationsProduceEncodingDirectivesThatModifySymbolTable(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 1), @@ -1659,11 +1692,14 @@ public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(InputT } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 1), @@ -1674,11 +1710,14 @@ public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAw ); } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void multiValuePartialMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void multiValuePartialMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(inputFormat); ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 2, false); verifyStream("Pi 3.14159".getBytes(StandardCharsets.UTF_8), out, outputFormat, @@ -1749,11 +1788,14 @@ public void emptyMacroAppendToEmptyTable(InputType inputType, StreamType streamT } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void emptyMacroAppendToEmptyTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = emptyMacroAppendToEmptyTable(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void emptyMacroAppendToEmptyTableMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = emptyMacroAppendToEmptyTable(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), @@ -1810,11 +1852,14 @@ public void emptyMacroAppendToNonEmptyTable(InputType inputType, StreamType stre } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void emptyMacroAppendToNonEmptyTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = emptyMacroAppendToNonEmptyTable(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void emptyMacroAppendToNonEmptyTableMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = emptyMacroAppendToNonEmptyTable(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), @@ -1872,11 +1917,14 @@ public void invokeUnqualifiedSystemMacroInTDL(InputType inputType, StreamType st } } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { - byte[] data = invokeUnqualifiedSystemMacroInTDL(StreamType.BINARY); + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { + byte[] data = invokeUnqualifiedSystemMacroInTDL(inputFormat); verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), @@ -1887,12 +1935,15 @@ public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode(InputType input ); } - // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void multipleIonVersionMarkersMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + @ParameterizedTest(name = "{0},{1},{2}") + @MethodSource("allInputFormatsInputTypesAndOutputFormats") + public void multipleIonVersionMarkersMacroAwareTranscode( + InputType inputType, + StreamType inputFormat, + StreamType outputFormat + ) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - IonRawWriter_1_1 writer = StreamType.BINARY.newWriter(out); + IonRawWriter_1_1 writer = inputFormat.newWriter(out); Map symbols = new HashMap<>(); writer.writeIVM(); writeSymbolTableAppendEExpression(writer, symbols, "foo"); diff --git a/src/test/java/com/amazon/ion/impl/IonRawTextReaderTest_1_1.java b/src/test/java/com/amazon/ion/impl/IonRawTextReaderTest_1_1.java index ebf8e73082..d164951064 100644 --- a/src/test/java/com/amazon/ion/impl/IonRawTextReaderTest_1_1.java +++ b/src/test/java/com/amazon/ion/impl/IonRawTextReaderTest_1_1.java @@ -139,12 +139,12 @@ public void invalidExpressionSyntax(int minorVersion, String input, String first "(:values (:: ) ) 0 1", "(:values (:: 0 1))", "(:values 0 1)", - "(:values (:: (:: 0) (:values (:: 1))))", - "(:values (:: 0) (:values (:: 1)))", + // "(:values (:: (:: 0) (:values (:: 1))))", // TODO make this illegal: https://github.com/amazon-ion/ion-java/issues/1009 + // "(:values (:: 0) (:values (:: 1)))", // TODO make this illegal: https://github.com/amazon-ion/ion-java/issues/1009 "(:values (:values (:: 0 1)))", "(:values (:values 0 1))", "(:1 (:1 0 1))", - "(:1 (:: (:: 0) (:1 (:: 1))))" + // "(:1 (:: (:: 0) (:1 (:: 1))))" // TODO make this illegal: https://github.com/amazon-ion/ion-java/issues/1009 }) public void validValuesInvocations(String text) throws Exception { try (IonReader reader = newTextReader("$ion_1_1 " + text)) {