class JSON::ResumableParser

Public Class Methods

Source

static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
{
    rb_check_arity(argc, 0, 1);
    rb_check_frozen(self);
    JSON_ResumableParser *parser = cResumableParser_get(self);

    VALUE opts = argc > 0 ? argv[0] : Qnil;
    parser_config_init(&parser->config, opts, self);

    return self;
}

Creates a new JSON::ResumableParser instance.

Argument opts, if given, contains a Hash of options for the parsing. See Parsing Options.

A ResumableParser is able to parse partial documents and resume parsing later when more of the document is provided:

parser = JSON::ResumableParser.new
parser << '{"user": "george", "role": "ad'
parser.parse # => false
parser.eos? # => true
parser.partial_value # => { "user" => "george", "role" => nil }
parser.rest # => '"ad'

parser << 'min" }[1, 2, 3]'
parser.parse # => true
parser.value # => { "user" => "george", "role" => "admin" }

parser.parse # => true
parser.value # => [1, 2, 3]

While ResumableParser is able to parse streams of documents without any explicit separators between them, it is highly recommended to separate documents by either spaces or newlines, as otherwise the JSON syntax for numbers may be ambiguous. When parsing a number, ResumableParser will not consider the number complete until something follows:

parser << '123'
parser.parse # => false
parser << ' '
parser.parse # => true
parser.value # => 123

Public Instance Methods

self << string → self

Source

static VALUE cResumableParser_feed(VALUE self, VALUE str)
{
    rb_check_frozen(self);
    str = convert_encoding(str);
    if (!RSTRING_LEN(str)) {
        return self;
    }

    JSON_ResumableParser *parser = cResumableParser_get(self);

    size_t offset = parser->state.cursor - parser->state.start;
    const size_t remaining = parser->state.end - parser->state.cursor;

    if (!remaining) {
        parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
        offset = 0;
    } else {
        JSON_ASSERT(parser->buffer);

        const size_t size = parser->state.end - parser->state.start;
        const size_t consumed = size - remaining;
        char *old_ptr = RSTRING_PTR(parser->buffer);

        if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
            VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
            MEMCPY(RSTRING_PTR(new_buffer), old_ptr + consumed, char, remaining);
            rb_str_set_len(new_buffer, remaining);
            offset = 0;
            parser->buffer = new_buffer;
        } else if (consumed > (size / 2) && size >= 512) {
            MEMMOVE(old_ptr, old_ptr + consumed, char, remaining);
            rb_str_set_len(parser->buffer, remaining);
            offset = 0;
        }
        rb_str_append(parser->buffer, str);
    }

    long len;
    const char *start;
    RSTRING_GETMEM(parser->buffer, start, len);
    parser->state.start = start;
    parser->state.end = start + len;
    parser->state.cursor = parser->state.start + offset;

    return self;
}

Appends the given string to the parser’s buffer.

clear → self

Source

static VALUE cResumableParser_clear(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
    parser->buffer = 0;
    parser->frames.head = 0;
    parser->value_stack.head = 0;
    parser->state.name_cache.length = 0;
    parser->state.start = parser->state.cursor = parser->state.end = NULL;
    return self;
}

Entirely reset the parser state and buffer.

value? → true or false

Source

static VALUE cResumableParser_eos_p(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);
    return eos(&parser->state) ? Qtrue : Qfalse;
}

Returns whether the internal buffer has been entirely consumed.

parse → true or false

Source

static VALUE cResumableParser_parse(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
    if (!parser->buffer) {
        return Qfalse;
    }

    if (parser->frames.head == 0) {
        json_frame_stack_push(&parser->state, (json_frame){
            .type = JSON_FRAME_ROOT,
            .phase = JSON_PHASE_VALUE,
        });
    }

    VALUE Vsource = parser->buffer; // Prevent compaction

    json_frame *frame = json_frame_stack_peek(&parser->frames);

    if (frame->phase == JSON_PHASE_DONE) {
        JSON_ASSERT(parser->value_stack.head == 1);
        JSON_ASSERT(parser->frames.head == 1);

        frame->phase = JSON_PHASE_VALUE;
        rvalue_stack_pop(parser->state.value_stack, 1);
    }

    struct json_parse_any_args args = {
        .state = &parser->state,
        .config = &parser->config,
    };
    int status;
    bool complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
    parser->in_use = false;
    if (status) {
        complete = false;
        if (RTEST(rb_ivar_get(rb_errinfo(), rb_intern("@eos")))) {
            complete = false; // is an EOS error
        } else {
            rb_jump_tag(status); // reraise
        }
    }
    RB_GC_GUARD(Vsource);
    return complete ? Qtrue : Qfalse;
}

Attemps to parse a JSON document from the internal buffer. Returns whether a complete document could be parsed.

It does raise JSON::ParserError when encountering invalid JSON syntax.

The parsed object can be retrieved by calling value

partial_value → object

Source

static VALUE cResumableParser_partial_value(VALUE self)
{
    JSON_ResumableParser *original_parser = ResumableParser_acquire(self, false);
    JSON_ResumableParser parser = *original_parser;

    parser.state.frames = &parser.frames;
    parser.state.value_stack = &parser.value_stack;

    if (parser.value_stack.head == 0) {
        return Qnil;
    }

    json_frame *frame = json_frame_stack_peek(parser.state.frames);
    long missing_object_value = 0;
    if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
        missing_object_value = 1;
    }

    // Copy the value stack as we need to mutate it.
    long capa = parser.value_stack.head;
    parser.value_stack.capa = (capa + missing_object_value);
    VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value);
    MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa);
    parser.value_stack.ptr = value_stack_buffer;

    JSON_ParserState *state = &parser.state;
    JSON_ParserConfig *config = &parser.config;

    if (missing_object_value) {
        rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
    }

    VALUE partial_result = Qundef;

    while (UNDEF_P(partial_result)) {
        frame = json_frame_stack_peek(state->frames);

        switch (frame->type) {
            case JSON_FRAME_ROOT: {
                partial_result = *rvalue_stack_peek(state->value_stack, 1);
                break;
            }

            case JSON_FRAME_ARRAY: {
                long count = json_frame_entry_count(frame, state->value_stack);
                json_push_value(state, config, json_decode_array(state, config, count));
                json_frame_stack_pop(state->frames);

                break;
            }

            case JSON_FRAME_OBJECT: {
                long count = json_frame_entry_count(frame, state->value_stack);
                json_push_value(state, config, json_decode_object(state, config, count));
                json_frame_stack_pop(state->frames);
                break;
            }

            default: {
                JSON_UNREACHABLE_RETURN(Qundef);
                break;
            }
        }
    }

    ALLOCV_END(tmpbuf);
    return partial_result;
}

Returns the Ruby objects parsed up to this point:

parser << '[1, [2, 3,'
parser.parse # => false
parser.value # ArgumentError no ready value
parser.partial_value # => [1, [2, 3]]

rest → string

Source

static VALUE cResumableParser_rest(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);

    if (!parser->buffer) {
        return rb_utf8_str_new("", 0);
    }

    size_t offset = parser->state.cursor - parser->state.start;
    const char *ptr;
    long len;
    RSTRING_GETMEM(parser->buffer, ptr, len);
    return rb_utf8_str_new(ptr + offset, len - offset);
}

Returns a string containing what remains to be parsed in the buffer

parser << '{ "message": "unterminated message'
parser.parse # => false
parser.rest # => '"unterminated message"'

value → object

Source

static VALUE cResumableParser_value(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);

    if (parser->frames.head > 0) {
        json_frame *frame = json_frame_stack_peek(&parser->frames);

        if (frame->phase == JSON_PHASE_DONE) {
            VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
            rvalue_stack_pop(parser->state.value_stack, 1);
            json_frame_stack_pop(parser->state.frames);
            return result;
        }
    }
    rb_raise(rb_eArgError, "no ready value");
}

Returns and consume the last parsed value. Raises ArgumentError if there is no parsed value or if it was already retrieved:

parser << '[1][2]'
parser.value # ArgumentError no ready value
parser.parse # => true
parser.value # => [1]
parser.value # ArgumentError no ready value

value? → true or false

Source

static VALUE cResumableParser_value_p(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);

    if (parser->value_stack.head > 0) {
        json_frame *frame = json_frame_stack_peek(&parser->frames);
        if (frame->phase == JSON_PHASE_DONE) {
            return Qtrue;
        }
    }
    return Qfalse;
}

Returns whether a parsed value is available.

class JSON::ResumableParser

Limitations