Skip to content

Commit

Permalink
Merge pull request #686 from byroot/io-flush
Browse files Browse the repository at this point in the history 
JSON.dump: write directly into the provided IO
  • Loading branch information
@byroot
byroot authored Nov 21, 2024
2 parents 2d62ec4 + 0824b62 commit 871e13e
Show file tree
Hide file tree
Showing 14 changed files with 279 additions and 164 deletions.
4 changes: 2 additions & 2 deletions Rakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby'
file JRUBY_PARSER_JAR => :compile do
cd 'java/src' do
parser_classes = FileList[
"json/ext/ByteListTranscoder*.class",
"json/ext/ByteList*.class",
"json/ext/OptionsReader*.class",
"json/ext/Parser*.class",
"json/ext/RuntimeInfo*.class",
Expand All @@ -179,7 +179,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby'
file JRUBY_GENERATOR_JAR => :compile do
cd 'java/src' do
generator_classes = FileList[
"json/ext/ByteListTranscoder*.class",
"json/ext/ByteList*.class",
"json/ext/OptionsReader*.class",
"json/ext/Generator*.class",
"json/ext/RuntimeInfo*.class",
Expand Down
62 changes: 46 additions & 16 deletions ext/json/ext/fbuffer/fbuffer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ typedef struct FBufferStruct {
unsigned long len;
unsigned long capa;
char *ptr;
VALUE io;
} FBuffer;

#define FBUFFER_STACK_SIZE 512
#define FBUFFER_IO_BUFFER_SIZE (16384 - 1)
#define FBUFFER_INITIAL_LENGTH_DEFAULT 1024

#define FBUFFER_PTR(fb) ((fb)->ptr)
Expand All @@ -66,7 +68,7 @@ static void fbuffer_append_long(FBuffer *fb, long number);
#endif
static inline void fbuffer_append_char(FBuffer *fb, char newchr);
#ifdef JSON_GENERATOR
static VALUE fbuffer_to_s(FBuffer *fb);
static VALUE fbuffer_finalize(FBuffer *fb);
#endif

static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char *stack_buffer, long stack_buffer_size)
Expand All @@ -86,24 +88,19 @@ static void fbuffer_free(FBuffer *fb)
}
}

#ifndef JSON_GENERATOR
static void fbuffer_clear(FBuffer *fb)
{
fb->len = 0;
}
#endif

static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested)
static void fbuffer_flush(FBuffer *fb)
{
unsigned long required;

if (RB_UNLIKELY(!fb->ptr)) {
fb->ptr = ALLOC_N(char, fb->initial_length);
fb->capa = fb->initial_length;
}

for (required = fb->capa; requested > required - fb->len; required <<= 1);
rb_io_write(fb->io, rb_utf8_str_new(fb->ptr, fb->len));
fbuffer_clear(fb);
}

static void fbuffer_realloc(FBuffer *fb, unsigned long required)
{
if (required > fb->capa) {
if (fb->type == FBUFFER_STACK_ALLOCATED) {
const char *old_buffer = fb->ptr;
Expand All @@ -117,6 +114,32 @@ static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested)
}
}

static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested)
{
if (RB_UNLIKELY(fb->io)) {
if (fb->capa < FBUFFER_IO_BUFFER_SIZE) {
fbuffer_realloc(fb, FBUFFER_IO_BUFFER_SIZE);
} else {
fbuffer_flush(fb);
}

if (RB_LIKELY(requested < fb->capa)) {
return;
}
}

unsigned long required;

if (RB_UNLIKELY(!fb->ptr)) {
fb->ptr = ALLOC_N(char, fb->initial_length);
fb->capa = fb->initial_length;
}

for (required = fb->capa; requested > required - fb->len; required <<= 1);

fbuffer_realloc(fb, required);
}

static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested)
{
if (RB_UNLIKELY(requested > fb->capa - fb->len)) {
Expand Down Expand Up @@ -174,11 +197,18 @@ static void fbuffer_append_long(FBuffer *fb, long number)
fbuffer_append(fb, buffer_end - len, len);
}

static VALUE fbuffer_to_s(FBuffer *fb)
static VALUE fbuffer_finalize(FBuffer *fb)
{
VALUE result = rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb));
fbuffer_free(fb);
return result;
if (fb->io) {
fbuffer_flush(fb);
fbuffer_free(fb);
rb_io_flush(fb->io);
return fb->io;
} else {
VALUE result = rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb));
fbuffer_free(fb);
return result;
}
}
#endif
#endif
49 changes: 23 additions & 26 deletions ext/json/ext/generator/generator.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ struct generate_json_data {
};

static VALUE cState_from_state_s(VALUE self, VALUE opts);
static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func);
static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io);
static void generate_json(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj);
static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj);
static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj);
Expand Down Expand Up @@ -453,7 +453,7 @@ static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_object);
return cState_partial_generate(Vstate, self, generate_json_object, Qfalse);
}

/*
Expand All @@ -467,7 +467,7 @@ static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) {
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_array);
return cState_partial_generate(Vstate, self, generate_json_array, Qfalse);
}

#ifdef RUBY_INTEGER_UNIFICATION
Expand All @@ -480,7 +480,7 @@ static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_integer);
return cState_partial_generate(Vstate, self, generate_json_integer, Qfalse);
}

#else
Expand All @@ -493,7 +493,7 @@ static VALUE mFixnum_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_fixnum);
return cState_partial_generate(Vstate, self, generate_json_fixnum, Qfalse);
}

/*
Expand All @@ -505,7 +505,7 @@ static VALUE mBignum_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_bignum);
return cState_partial_generate(Vstate, self, generate_json_bignum, Qfalse);
}
#endif

Expand All @@ -518,7 +518,7 @@ static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_float);
return cState_partial_generate(Vstate, self, generate_json_float, Qfalse);
}

/*
Expand All @@ -543,7 +543,7 @@ static VALUE mString_to_json(int argc, VALUE *argv, VALUE self)
{
rb_check_arity(argc, 0, 1);
VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
return cState_partial_generate(Vstate, self, generate_json_string);
return cState_partial_generate(Vstate, self, generate_json_string, Qfalse);
}

/*
Expand Down Expand Up @@ -638,7 +638,7 @@ static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self)
rb_scan_args(argc, argv, "01", &state);
Check_Type(string, T_STRING);
state = cState_from_state_s(cState, state);
return cState_partial_generate(state, string, generate_json_string);
return cState_partial_generate(state, string, generate_json_string, Qfalse);
}

static void State_mark(void *ptr)
Expand Down Expand Up @@ -1045,12 +1045,14 @@ static VALUE generate_json_rescue(VALUE d, VALUE exc)
return Qundef;
}

static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func)
static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io)
{
GET_STATE(self);

char stack_buffer[FBUFFER_STACK_SIZE];
FBuffer buffer = {0};
FBuffer buffer = {
.io = RTEST(io) ? io : Qfalse,
};
fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE);

struct generate_json_data data = {
Expand All @@ -1062,19 +1064,12 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func)
};
rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);

return fbuffer_to_s(&buffer);
return fbuffer_finalize(&buffer);
}

/*
* call-seq: generate(obj)
*
* Generates a valid JSON document from object +obj+ and returns the
* result. If no valid JSON document can be created this method raises a
* GeneratorError exception.
*/
static VALUE cState_generate(VALUE self, VALUE obj)
static VALUE cState_generate(VALUE self, VALUE obj, VALUE io)
{
VALUE result = cState_partial_generate(self, obj, generate_json);
VALUE result = cState_partial_generate(self, obj, generate_json, io);
GET_STATE(self);
(void)state;
return result;
Expand Down Expand Up @@ -1502,14 +1497,16 @@ static VALUE cState_configure(VALUE self, VALUE opts)
return self;
}

static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts)
static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
{
JSON_Generator_State state = {0};
state_init(&state);
configure_state(&state, opts);

char stack_buffer[FBUFFER_STACK_SIZE];
FBuffer buffer = {0};
FBuffer buffer = {
.io = RTEST(io) ? io : Qfalse,
};
fbuffer_stack_init(&buffer, state.buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE);

struct generate_json_data data = {
Expand All @@ -1521,7 +1518,7 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts)
};
rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);

return fbuffer_to_s(&buffer);
return fbuffer_finalize(&buffer);
}

/*
Expand Down Expand Up @@ -1583,9 +1580,9 @@ void Init_generator(void)
rb_define_method(cState, "depth=", cState_depth_set, 1);
rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
rb_define_method(cState, "generate", cState_generate, 1);
rb_define_private_method(cState, "_generate", cState_generate, 2);

rb_define_singleton_method(cState, "generate", cState_m_generate, 2);
rb_define_singleton_method(cState, "generate", cState_m_generate, 3);

VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");

Expand Down
16 changes: 16 additions & 0 deletions java/src/json/ext/ByteListDirectOutputStream.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package json.ext;

import org.jcodings.Encoding;
import org.jruby.util.ByteList;

import java.io.ByteArrayOutputStream;

public class ByteListDirectOutputStream extends ByteArrayOutputStream {
ByteListDirectOutputStream(int size) {
super(size);
}

public ByteList toByteListDirect(Encoding encoding) {
return new ByteList(buf, 0, count, encoding, false);
}
}
21 changes: 12 additions & 9 deletions java/src/json/ext/ByteListTranscoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
import org.jruby.runtime.ThreadContext;
import org.jruby.util.ByteList;

import java.io.IOException;
import java.io.OutputStream;

/**
* A class specialized in transcoding a certain String format into another,
* using UTF-8 ByteLists as both input and output.
Expand All @@ -23,7 +26,7 @@ abstract class ByteListTranscoder {
/** Position of the next character to read */
protected int pos;

private ByteList out;
private OutputStream out;
/**
* When a character that can be copied straight into the output is found,
* its index is stored on this variable, and copying is delayed until
Expand All @@ -37,11 +40,11 @@ protected ByteListTranscoder(ThreadContext context) {
this.context = context;
}

protected void init(ByteList src, ByteList out) {
protected void init(ByteList src, OutputStream out) {
this.init(src, 0, src.length(), out);
}

protected void init(ByteList src, int start, int end, ByteList out) {
protected void init(ByteList src, int start, int end, OutputStream out) {
this.src = src;
this.pos = start;
this.charStart = start;
Expand Down Expand Up @@ -142,19 +145,19 @@ protected void quoteStart() {
* recently read character, or {@link #charStart} to quote
* until the character before it.
*/
protected void quoteStop(int endPos) {
protected void quoteStop(int endPos) throws IOException {
if (quoteStart != -1) {
out.append(src, quoteStart, endPos - quoteStart);
out.write(src.bytes(), quoteStart, endPos - quoteStart);
quoteStart = -1;
}
}

protected void append(int b) {
out.append(b);
protected void append(int b) throws IOException {
out.write(b);
}

protected void append(byte[] origin, int start, int length) {
out.append(origin, start, length);
protected void append(byte[] origin, int start, int length) throws IOException {
out.write(origin, start, length);
}


Expand Down
Loading

0 comments on commit 871e13e

Please sign in to comment.