Recycle toml token buffers

toml/LICENSE

@@ -1,5 +1,7 @@
 Jackson is licensed under [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt).
 
+---
+
 The TOML grammar and some parts of the test suite are adapted from the TOML project, which is licensed under the MIT license:
 
 The MIT License
@@ -23,3 +25,41 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+---
+
+The JFlex lexer generator is licensed under a 3-clause BSD-style license:
+
+JFlex - Copying, Warranty & License
+===================================
+
+JFlex is free software, since version 1.5 published under the terms of this
+[3-clause BSD-style license](https://opensource.org/licenses/BSD-3-Clause).
+
+There is absolutely NO WARRANTY for JFlex, its code and its documentation.
+
+
+Copyright (c) Gerwin Klein, Steve Rowe, Régis Décamps, Google LLC.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted
+provided that the following conditions are met:
+
+  * Redistributions of source code must retain the above copyright notice, this list of conditions
+    and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice, this list of
+    conditions and the following disclaimer in the documentation and/or other materials provided
+    with the distribution.
+  * Neither the names of the authors nor the names of JFlex contributors may be used to endorse
+    or promote products derived from this software without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

toml/pom.xml

@@ -71,6 +71,7 @@
                 <configuration>
                     <backup>false</backup>
                     <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>
+                    <skeleton>${project.basedir}/src/main/jflex/skeleton-toml</skeleton>
                 </configuration>
             </plugin>
         </plugins>

toml/src/main/java/com/fasterxml/jackson/dataformat/toml/Parser.java

@@ -1,5 +1,6 @@
 package com.fasterxml.jackson.dataformat.toml;
 
+import com.fasterxml.jackson.core.io.IOContext;
 import com.fasterxml.jackson.core.io.NumberInput;
 import com.fasterxml.jackson.core.util.VersionUtil;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -27,15 +28,28 @@ class Parser {
 
     private TomlToken next;
 
-    private Parser(JacksonTomlParseException.ErrorContext errorContext, int options, Reader reader) throws IOException {
+    private Parser(
+            IOContext ioContext,
+            JacksonTomlParseException.ErrorContext errorContext,
+            int options,
+            Reader reader
+    ) throws IOException {
         this.errorContext = errorContext;
         this.options = options;
-        this.lexer = new Lexer(reader, errorContext);
+        this.lexer = new Lexer(reader, ioContext, errorContext);
+        lexer.prohibitInternalBufferAllocate = (options & TomlWriteFeature.INTERNAL_PROHIBIT_INTERNAL_BUFFER_ALLOCATE) != 0;
         this.next = lexer.yylex();
     }
 
-    public static ObjectNode parse(JacksonTomlParseException.ErrorContext errorContext, int options, Reader reader) throws IOException {
-        return new Parser(errorContext, options, reader).parse();
+    public static ObjectNode parse(
+            IOContext ioContext,
+            int options,
+            Reader reader
+    ) throws IOException {
+        Parser parser = new Parser(ioContext, new JacksonTomlParseException.ErrorContext(ioContext, null), options, reader);
+        ObjectNode node = parser.parse();
+        parser.lexer.releaseBuffers();
+        return node;
     }
 
     private TomlToken peek() throws JacksonTomlParseException {

toml/src/main/java/com/fasterxml/jackson/dataformat/toml/TomlFactory.java

@@ -308,13 +308,12 @@ protected JsonGenerator _createUTF8Generator(OutputStream out, IOContext ctxt) t
      */
 
     private ObjectNode parse(IOContext ctxt, Reader r0) throws IOException {
-        JacksonTomlParseException.ErrorContext errorContext = new JacksonTomlParseException.ErrorContext(ctxt.getSourceReference(), null);
         if (ctxt.isResourceManaged() || isEnabled(StreamReadFeature.AUTO_CLOSE_SOURCE)) {
             try (Reader r = r0) {
-                return Parser.parse(errorContext, _tomlParserFeatures, r);
+                return Parser.parse(ctxt, _tomlParserFeatures, r);
             }
         } else {
-            return Parser.parse(errorContext, _tomlParserFeatures, r0);
+            return Parser.parse(ctxt, _tomlParserFeatures, r0);
         }
     }
 }

toml/src/main/java/com/fasterxml/jackson/dataformat/toml/TomlWriteFeature.java

@@ -15,6 +15,11 @@ public enum TomlWriteFeature implements FormatFeature {
      */
     FAIL_ON_NULL_WRITE(false);
 
+    /**
+     * Internal option for unit tests: Prohibit allocating internal buffers, except through the buffer recycler
+     */
+    static final int INTERNAL_PROHIBIT_INTERNAL_BUFFER_ALLOCATE = 0x80000000;
+
     final boolean _defaultState;
     final int _mask;
 

toml/src/main/jflex/com/fasterxml/jackson/dataformat/toml/toml.jflex

@@ -10,19 +10,48 @@ package com.fasterxml.jackson.dataformat.toml;
 %char
 %buffer 4000
 
+%ctorarg com.fasterxml.jackson.core.io.IOContext ioContext
 %ctorarg JacksonTomlParseException.ErrorContext errorContext
 
 %init{
+this.ioContext = ioContext;
 this.errorContext = errorContext;
 yybegin(EXPECT_EXPRESSION);
+this.zzBuffer = ioContext.allocTokenBuffer();
 %init}
 
 %{
+  private final com.fasterxml.jackson.core.io.IOContext ioContext;
   private final JacksonTomlParseException.ErrorContext errorContext;
 
+  boolean prohibitInternalBufferAllocate = false;
+  private boolean releaseTokenBuffer = true;
+
   private boolean trimmedNewline;
   StringBuilder stringBuilder = new StringBuilder();
 
+  private void requestLargerBuffer() throws JacksonTomlParseException {
+      if (prohibitInternalBufferAllocate) {
+          throw errorContext.atPosition(this).generic("Token too long, but buffer resizing prohibited");
+      }
+
+      // todo: use recycler
+      char[] newBuffer = new char[zzBuffer.length * 2];
+      System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
+      if (releaseTokenBuffer) {
+          ioContext.releaseTokenBuffer(zzBuffer);
+          releaseTokenBuffer = false;
+      }
+      zzBuffer = newBuffer;
+  }
+
+  public void releaseBuffers() {
+      if (releaseTokenBuffer) {
+          ioContext.releaseTokenBuffer(zzBuffer);
+          zzBuffer = null;
+      }
+  }
+
   private void startString() {
       stringBuilder.setLength(0);
       trimmedNewline = false;