transcode.c (11ce3778c6eacc10897729314e5ab3bed7830971)

/**********************************************************************
 
  transcode.c -
 
  $Author$
  created at: Tue Oct 30 16:10:22 JST 2007
 
  Copyright (C) 2007 Martin Duerst
 
**********************************************************************/
 
#include "ruby/internal/config.h"
 
#include <ctype.h>
 
#include "internal.h"
#include "internal/array.h"
#include "internal/inits.h"
#include "internal/gc.h"
#include "internal/object.h"
#include "internal/string.h"
#include "internal/transcode.h"
#include "internal/encoding.h"
#include "ruby/encoding.h"
#include "vm_sync.h"
 
#include "transcode_data.h"
#include "id.h"
 
#define ENABLE_ECONV_NEWLINE_OPTION 1
 
/* VALUE rb_cEncoding = rb_define_class("Encoding", rb_cObject); */
static VALUE rb_eUndefinedConversionError;
static VALUE rb_eInvalidByteSequenceError;
static VALUE rb_eConverterNotFoundError;
 
VALUE rb_cEncodingConverter;
 
static ID id_destination_encoding;
static ID id_destination_encoding_name;
static ID id_error_bytes;
static ID id_error_char;
static ID id_incomplete_input;
static ID id_readagain_bytes;
static ID id_source_encoding;
static ID id_source_encoding_name;
 
static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
static VALUE sym_xml, sym_text, sym_attr;
static VALUE sym_universal_newline;
static VALUE sym_crlf_newline;
static VALUE sym_cr_newline;
static VALUE sym_lf_newline;
#ifdef ENABLE_ECONV_NEWLINE_OPTION
static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
#endif
static VALUE sym_partial_input;
 
static VALUE sym_invalid_byte_sequence;
static VALUE sym_undefined_conversion;
static VALUE sym_destination_buffer_full;
static VALUE sym_source_buffer_empty;
static VALUE sym_finished;
static VALUE sym_after_output;
static VALUE sym_incomplete_input;
 
static unsigned char *
allocate_converted_string(const char *sname, const char *dname,
        const unsigned char *str, size_t len,
        unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
        size_t *dst_len_ptr, size_t *dst_bufsize_ptr);
 
/* dynamic structure, one per conversion (similar to iconv_t) */
/* may carry conversion state (e.g. for iso-2022-jp) */
typedef struct rb_transcoding {
    const rb_transcoder *transcoder;
 
    int flags;
 
    int resume_position;
    unsigned int next_table;
    VALUE next_info;
    unsigned char next_byte;
    unsigned int output_index;
 
    ssize_t recognized_len; /* already interpreted */
    ssize_t readagain_len; /* not yet interpreted */
    union {
        unsigned char ary[8]; /* max_input <= sizeof(ary) */
        unsigned char *ptr; /* length: max_input */
    } readbuf; /* recognized_len + readagain_len used */
 
    ssize_t writebuf_off;
    ssize_t writebuf_len;
    union {
        unsigned char ary[8]; /* max_output <= sizeof(ary) */
        unsigned char *ptr; /* length: max_output */
    } writebuf;
 
    union rb_transcoding_state_t { /* opaque data for stateful encoding */
        void *ptr;
        char ary[sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*)];
        double dummy_for_alignment;
    } state;
} rb_transcoding;
#define TRANSCODING_READBUF(tc) \
    ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
     (tc)->readbuf.ary : \
     (tc)->readbuf.ptr)
#define TRANSCODING_WRITEBUF(tc) \
    ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
     (tc)->writebuf.ary : \
     (tc)->writebuf.ptr)
#define TRANSCODING_WRITEBUF_SIZE(tc) \
    ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
     sizeof((tc)->writebuf.ary) : \
     (size_t)(tc)->transcoder->max_output)
#define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
#define TRANSCODING_STATE(tc) \
    ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
     (tc)->state.ary : \
     (tc)->state.ptr)
 
typedef struct {
    struct rb_transcoding *tc;
    unsigned char *out_buf_start;
    unsigned char *out_data_start;
    unsigned char *out_data_end;
    unsigned char *out_buf_end;
    rb_econv_result_t last_result;
} rb_econv_elem_t;
 
struct rb_econv_t {
    int flags;
    int started; /* bool */
 
    const char *source_encoding_name;
    const char *destination_encoding_name;
 
    const unsigned char *replacement_str;
    size_t replacement_len;
    size_t replacement_bufsize;
    const char *replacement_enc;
 
    unsigned char *in_buf_start;
    unsigned char *in_data_start;
    unsigned char *in_data_end;
    unsigned char *in_buf_end;
    rb_econv_elem_t *elems;
    int replacement_allocated; /* bool */
    int num_allocated;
    int num_trans;
    int num_finished;
    struct rb_transcoding *last_tc;
 
    /* last error */
    struct {
        rb_econv_result_t result;
        struct rb_transcoding *error_tc;
        const char *source_encoding;
        const char *destination_encoding;
        const unsigned char *error_bytes_start;
        size_t error_bytes_len;
        size_t readagain_len;
    } last_error;
 
    /* The following fields are only for Encoding::Converter.
     * rb_econv_open set them NULL. */
    rb_encoding *source_encoding;
    rb_encoding *destination_encoding;
};
 
/*
 *  Dispatch data and logic
 */
 
#define DECORATOR_P(sname, dname) (*(sname) == '\0')
 
typedef struct {
    const char *sname;
    const char *dname;
    const char *lib; /* null means no need to load a library */
    const rb_transcoder *transcoder;
} transcoder_entry_t;
 
static st_table *transcoder_table;
 
static int
free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
{
    SIZED_FREE((transcoder_entry_t *)val);
    return ST_DELETE;
}
 
static int
free_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
{
    st_foreach((void *)val, free_inner_transcode_i, 0);
    st_free_table((void *)val);
    return ST_DELETE;
}
 
void
rb_free_transcoder_table(void)
{
    st_foreach(transcoder_table, free_transcode_i, 0);
    st_free_table(transcoder_table);
}
 
static transcoder_entry_t *
make_transcoder_entry(const char *sname, const char *dname)
{
    st_data_t val;
    st_table *table2;
 
    RB_VM_LOCKING() {
        if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
            val = (st_data_t)st_init_strcasetable();
            st_add_direct(transcoder_table, (st_data_t)sname, val);
        }
        table2 = (st_table *)val;
        if (!st_lookup(table2, (st_data_t)dname, &val)) {
            transcoder_entry_t *entry = ALLOC(transcoder_entry_t);
            entry->sname = sname;
            entry->dname = dname;
            entry->lib = NULL;
            entry->transcoder = NULL;
            val = (st_data_t)entry;
            st_add_direct(table2, (st_data_t)dname, val);
        }
    }
    return (transcoder_entry_t *)val;
}
 
static transcoder_entry_t *
get_transcoder_entry(const char *sname, const char *dname)
{
    st_data_t val = 0;
    st_table *table2;
    RB_VM_LOCKING() {
        if (st_lookup(transcoder_table, (st_data_t)sname, &val)) {
            table2 = (st_table *)val;
            if (!st_lookup(table2, (st_data_t)dname, &val)) {
                val = 0;
            }
        }
    }
    return (transcoder_entry_t *)val;
}
 
void
rb_register_transcoder(const rb_transcoder *tr)
{
    const char *const sname = tr->src_encoding;
    const char *const dname = tr->dst_encoding;
 
    transcoder_entry_t *entry;
 
    RB_VM_LOCKING() {
        entry = make_transcoder_entry(sname, dname);
        if (entry->transcoder) {
            rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered",
                    sname, dname);
        }
        entry->transcoder = tr;
    }
}
 
static void
declare_transcoder(const char *sname, const char *dname, const char *lib)
{
    transcoder_entry_t *entry;
 
    entry = make_transcoder_entry(sname, dname);
    entry->lib = lib;
}
 
static const char transcoder_lib_prefix[] = "enc/trans/";
 
void
rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
{
    if (!lib) {
        rb_raise(rb_eArgError, "invalid library name - (null)");
    }
    declare_transcoder(enc1, enc2, lib);
}
 
#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
 
typedef struct search_path_queue_tag {
    struct search_path_queue_tag *next;
    const char *enc;
} search_path_queue_t;
 
typedef struct {
    st_table *visited;
    search_path_queue_t *queue;
    search_path_queue_t **queue_last_ptr;
    const char *base_enc;
} search_path_bfs_t;
 
static int
transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
{
    const char *dname = (const char *)key;
    search_path_bfs_t *bfs = (search_path_bfs_t *)arg;
    search_path_queue_t *q;
 
    if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
        return ST_CONTINUE;
    }
 
    q = ALLOC(search_path_queue_t);
    q->enc = dname;
    q->next = NULL;
    *bfs->queue_last_ptr = q;
    bfs->queue_last_ptr = &q->next;
 
    st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
    return ST_CONTINUE;
}
 
static int
transcode_search_path(const char *sname, const char *dname,
    void (*callback)(const char *sname, const char *dname, int depth, void *arg),
    void *arg)
{
    search_path_bfs_t bfs;
    search_path_queue_t *q;
    st_data_t val;
    st_table *table2;
    int pathlen = -1;
    bool found = false;
    bool lookup_res;
 
    if (encoding_equal(sname, dname))
        return -1;
 
    q = ALLOC(search_path_queue_t);
    q->enc = sname;
    q->next = NULL;
    bfs.queue_last_ptr = &q->next;
    bfs.queue = q;
 
    bfs.visited = st_init_strcasetable(); // due to base encodings, we need to do search in a loop
    st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
 
    RB_VM_LOCKING() {
        while (bfs.queue) {
            q = bfs.queue;
            bfs.queue = q->next;
            if (!bfs.queue) {
                bfs.queue_last_ptr = &bfs.queue;
            }
 
            lookup_res = st_lookup(transcoder_table, (st_data_t)q->enc, &val); // src => table2
            if (!lookup_res) {
                SIZED_FREE(q);
                continue;
            }
            table2 = (st_table *)val;
 
            if (st_lookup(table2, (st_data_t)dname, &val)) { // dest => econv
                st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
                SIZED_FREE(q);
                found = true;
                break;
            }
 
            bfs.base_enc = q->enc;
            st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
 
            bfs.base_enc = NULL;
            SIZED_FREE(q);
        }
    }
 
    while (bfs.queue) {
        q = bfs.queue;
        bfs.queue = q->next;
        SIZED_FREE(q);
    }
 
    if (found) {
        const char *enc = dname;
        int depth;
        pathlen = 0;
        while (1) {
            st_lookup(bfs.visited, (st_data_t)enc, &val);
            if (!val)
                break;
            pathlen++;
            enc = (const char *)val;
        }
        depth = pathlen;
        enc = dname;
        while (1) {
            st_lookup(bfs.visited, (st_data_t)enc, &val);
            if (!val)
                break;
            callback((const char *)val, enc, --depth, arg);
            enc = (const char *)val;
        }
    }
 
    st_free_table(bfs.visited);
 
    return pathlen; /* is -1 if not found */
}
 
int rb_require_internal_silent(VALUE fname);
 
static const rb_transcoder *
load_transcoder_entry(transcoder_entry_t *entry)
{
    ASSERT_vm_unlocking();
    if (entry->transcoder)
        return entry->transcoder;
 
    if (entry->lib) {
        const char *const lib = entry->lib;
        const size_t len = strlen(lib);
        const size_t total_len = sizeof(transcoder_lib_prefix) - 1 + len;
        const VALUE fn = rb_str_new(0, total_len);
        char *const path = RSTRING_PTR(fn);
 
        memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1);
        memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len);
        rb_str_set_len(fn, total_len);
        OBJ_FREEZE(fn);
        rb_require_internal_silent(fn); // Sets entry->transcoder
    }
 
    if (entry->transcoder)
        return entry->transcoder;
 
    return NULL;
}
 
static const char*
get_replacement_character(const char *encname, size_t *len_ret, const char **repl_encname_ptr)
{
    if (encoding_equal(encname, "UTF-8")) {
        *len_ret = 3;
        *repl_encname_ptr = "UTF-8";
        return "\xEF\xBF\xBD";
    }
    else {
        *len_ret = 1;
        *repl_encname_ptr = "US-ASCII";
        return "?";
    }
}
 
/*
 *  Transcoding engine logic
 */
 
static const unsigned char *
transcode_char_start(rb_transcoding *tc,
                         const unsigned char *in_start,
                         const unsigned char *inchar_start,
                         const unsigned char *in_p,
                         size_t *char_len_ptr)
{
    const unsigned char *ptr;
    if (inchar_start - in_start < tc->recognized_len) {
        MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
               inchar_start, unsigned char, in_p - inchar_start);
        ptr = TRANSCODING_READBUF(tc);
    }
    else {
        ptr = inchar_start - tc->recognized_len;
    }
    *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
    return ptr;
}
 
static rb_econv_result_t
transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
                      const unsigned char *in_stop, unsigned char *out_stop,
                      rb_transcoding *tc,
                      const int opt)
{
    const rb_transcoder *tr = tc->transcoder;
    int unitlen = tr->input_unit_length;
    ssize_t readagain_len = 0;
 
    const unsigned char *inchar_start;
    const unsigned char *in_p;
 
    unsigned char *out_p;
 
    in_p = inchar_start = *in_pos;
 
    out_p = *out_pos;
 
#define SUSPEND(ret, num) \
    do { \
        tc->resume_position = (num); \
        if (0 < in_p - inchar_start) \
            MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
                   inchar_start, unsigned char, in_p - inchar_start); \
        *in_pos = in_p; \
        *out_pos = out_p; \
        tc->recognized_len += in_p - inchar_start; \
        if (readagain_len) { \
            tc->recognized_len -= readagain_len; \
            tc->readagain_len = readagain_len; \
        } \
        return (ret); \
        resume_label ## num:; \
    } while (0)
#define SUSPEND_OBUF(num) \
    do { \
        while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
    } while (0)
 
#define SUSPEND_AFTER_OUTPUT(num) \
    if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
        SUSPEND(econv_after_output, num); \
    }
 
#define next_table (tc->next_table)
#define next_info (tc->next_info)
#define next_byte (tc->next_byte)
#define writebuf_len (tc->writebuf_len)
#define writebuf_off (tc->writebuf_off)
 
    switch (tc->resume_position) {
      case 0: break;
      case 1: goto resume_label1;
      case 2: goto resume_label2;
      case 3: goto resume_label3;
      case 4: goto resume_label4;
      case 5: goto resume_label5;
      case 6: goto resume_label6;
      case 7: goto resume_label7;
      case 8: goto resume_label8;
      case 9: goto resume_label9;
      case 10: goto resume_label10;
      case 11: goto resume_label11;
      case 12: goto resume_label12;
      case 13: goto resume_label13;
      case 14: goto resume_label14;
      case 15: goto resume_label15;
      case 16: goto resume_label16;
      case 17: goto resume_label17;
      case 18: goto resume_label18;
      case 19: goto resume_label19;
      case 20: goto resume_label20;
      case 21: goto resume_label21;
      case 22: goto resume_label22;
      case 23: goto resume_label23;
      case 24: goto resume_label24;
      case 25: goto resume_label25;
      case 26: goto resume_label26;
      case 27: goto resume_label27;
      case 28: goto resume_label28;
      case 29: goto resume_label29;
      case 30: goto resume_label30;
      case 31: goto resume_label31;
      case 32: goto resume_label32;
      case 33: goto resume_label33;
      case 34: goto resume_label34;
    }
 
    while (1) {
        inchar_start = in_p;
        tc->recognized_len = 0;
        next_table = tr->conv_tree_start;
 
        SUSPEND_AFTER_OUTPUT(24);
 
        if (in_stop <= in_p) {
            if (!(opt & ECONV_PARTIAL_INPUT))
                break;
            SUSPEND(econv_source_buffer_empty, 7);
            continue;
        }
 
#define BYTE_ADDR(index) (tr->byte_array + (index))
#define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
#define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
#define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
#define BL_MIN_BYTE     (BL_BASE[0])
#define BL_MAX_BYTE     (BL_BASE[1])
#define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
#define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
 
        next_byte = (unsigned char)*in_p++;
      follow_byte:
        if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
            next_info = INVALID;
        else {
            next_info = (VALUE)BL_ACTION(next_byte);
        }
      follow_info:
        switch (next_info & 0x1F) {
          case NOMAP:
            {
                const unsigned char *p = inchar_start;
                writebuf_off = 0;
                while (p < in_p) {
                    TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (unsigned char)*p++;
                }
                writebuf_len = writebuf_off;
                writebuf_off = 0;
                while (writebuf_off < writebuf_len) {
                    SUSPEND_OBUF(3);
                    *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
                }
            }
            continue;
          case 0x00: case 0x04: case 0x08: case 0x0C:
          case 0x10: case 0x14: case 0x18: case 0x1C:
            SUSPEND_AFTER_OUTPUT(25);
            while (in_p >= in_stop) {
                if (!(opt & ECONV_PARTIAL_INPUT))
                    goto incomplete;
                SUSPEND(econv_source_buffer_empty, 5);
            }
            next_byte = (unsigned char)*in_p++;
            next_table = (unsigned int)next_info;
            goto follow_byte;
          case ZERObt: /* drop input */
            continue;
          case ONEbt:
            SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
            continue;
          case TWObt:
            SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
            SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
            continue;
          case THREEbt:
            SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
            SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
            SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
            continue;
          case FOURbt:
            SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
            SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
            SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
            SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
            continue;
          case GB4bt:
            SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
            SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
            SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
            SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
            continue;
          case STR1:
            tc->output_index = 0;
            while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
                SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
                tc->output_index++;
            }
            continue;
          case FUNii:
            next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info);
            goto follow_info;
          case FUNsi:
            {
                const unsigned char *char_start;
                size_t char_len;
                char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
                next_info = (VALUE)(*tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
                goto follow_info;
            }
          case FUNio:
            SUSPEND_OBUF(13);
            if (tr->max_output <= out_stop - out_p)
                out_p += tr->func_io(TRANSCODING_STATE(tc),
                    next_info, out_p, out_stop - out_p);
            else {
                writebuf_len = tr->func_io(TRANSCODING_STATE(tc),
                    next_info,
                    TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
                writebuf_off = 0;
                while (writebuf_off < writebuf_len) {
                    SUSPEND_OBUF(20);
                    *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
                }
            }
            break;
          case FUNso:
            {
                const unsigned char *char_start;
                size_t char_len;
                SUSPEND_OBUF(14);
                if (tr->max_output <= out_stop - out_p) {
                    char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
                    out_p += tr->func_so(TRANSCODING_STATE(tc),
                        char_start, (size_t)char_len,
                        out_p, out_stop - out_p);
                }
                else {
                    char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
                    writebuf_len = tr->func_so(TRANSCODING_STATE(tc),
                        char_start, (size_t)char_len,
                        TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
                    writebuf_off = 0;
                    while (writebuf_off < writebuf_len) {
                        SUSPEND_OBUF(22);
                        *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
                    }
                }
                break;
            }
          case FUNsio:
            {
                const unsigned char *char_start;
                size_t char_len;
                SUSPEND_OBUF(33);
                if (tr->max_output <= out_stop - out_p) {
                    char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
                    out_p += tr->func_sio(TRANSCODING_STATE(tc),
                        char_start, (size_t)char_len, next_info,
                        out_p, out_stop - out_p);
                }
                else {
                    char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
                    writebuf_len = tr->func_sio(TRANSCODING_STATE(tc),
                        char_start, (size_t)char_len, next_info,
                        TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
                    writebuf_off = 0;
                    while (writebuf_off < writebuf_len) {
                        SUSPEND_OBUF(34);
                        *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
                    }
                }
                break;
            }
          case INVALID:
            if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
                if (tc->recognized_len + (in_p - inchar_start) < unitlen)
                    SUSPEND_AFTER_OUTPUT(26);
                while ((opt & ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
                    in_p = in_stop;
                    SUSPEND(econv_source_buffer_empty, 8);
                }
                if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
                    in_p = in_stop;
                }
                else {
                    in_p = inchar_start + (unitlen - tc->recognized_len);
                }
            }
            else {
                ssize_t invalid_len; /* including the last byte which causes invalid */
                ssize_t discard_len;
                invalid_len = tc->recognized_len + (in_p - inchar_start);
                discard_len = ((invalid_len - 1) / unitlen) * unitlen;
                readagain_len = invalid_len - discard_len;
            }
            goto invalid;
          case UNDEF:
            goto undef;
          default:
            rb_raise(rb_eRuntimeError, "unknown transcoding instruction");
        }
        continue;
 
      invalid:
        SUSPEND(econv_invalid_byte_sequence, 1);
        continue;
 
      incomplete:
        SUSPEND(econv_incomplete_input, 27);
        continue;
 
      undef:
        SUSPEND(econv_undefined_conversion, 2);
        continue;
    }
 
    /* cleanup */
    if (tr->finish_func) {
        SUSPEND_OBUF(4);
        if (tr->max_output <= out_stop - out_p) {
            out_p += tr->finish_func(TRANSCODING_STATE(tc),
                out_p, out_stop - out_p);
        }
        else {
            writebuf_len = tr->finish_func(TRANSCODING_STATE(tc),
                TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
            writebuf_off = 0;
            while (writebuf_off < writebuf_len) {
                SUSPEND_OBUF(23);
                *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
            }
        }
    }
    while (1)
        SUSPEND(econv_finished, 6);
#undef SUSPEND
#undef next_table
#undef next_info
#undef next_byte
#undef writebuf_len
#undef writebuf_off
}
 
static rb_econv_result_t
transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos,
                      const unsigned char *in_stop, unsigned char *out_stop,
                      rb_transcoding *tc,
                      const int opt)
{
    if (tc->readagain_len) {
        unsigned char *readagain_buf = ALLOCA_N(unsigned char, tc->readagain_len);
        const unsigned char *readagain_pos = readagain_buf;
        const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
        rb_econv_result_t res;
 
        MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
               unsigned char, tc->readagain_len);
        tc->readagain_len = 0;
        res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|ECONV_PARTIAL_INPUT);
        if (res != econv_source_buffer_empty) {
            MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
                   readagain_pos, unsigned char, readagain_stop - readagain_pos);
            tc->readagain_len += readagain_stop - readagain_pos;
            return res;
        }
    }
    return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
}
 
static rb_transcoding *
rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags)
{
    rb_transcoding *tc;
 
    tc = ALLOC(rb_transcoding);
    tc->transcoder = tr;
    tc->flags = flags;
    if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
        tc->state.ptr = xmalloc(tr->state_size);
    if (tr->state_init_func) {
        (tr->state_init_func)(TRANSCODING_STATE(tc)); /* xxx: check return value */
    }
    tc->resume_position = 0;
    tc->recognized_len = 0;
    tc->readagain_len = 0;
    tc->writebuf_len = 0;
    tc->writebuf_off = 0;
    if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
        tc->readbuf.ptr = xmalloc(tr->max_input);
    }
    if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
        tc->writebuf.ptr = xmalloc(tr->max_output);
    }
    return tc;
}
 
static rb_econv_result_t
rb_transcoding_convert(rb_transcoding *tc,
  const unsigned char **input_ptr, const unsigned char *input_stop,
  unsigned char **output_ptr, unsigned char *output_stop,
  int flags)
{
    return transcode_restartable(
                input_ptr, output_ptr,
                input_stop, output_stop,
                tc, flags);
}
 
static void
rb_transcoding_close(rb_transcoding *tc)
{
    const rb_transcoder *tr = tc->transcoder;
    if (tr->state_fini_func) {
        (tr->state_fini_func)(TRANSCODING_STATE(tc)); /* check return value? */
    }
    if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
        ruby_xfree_sized(tc->state.ptr, tr->state_size);
    if ((int)sizeof(tc->readbuf.ary) < tr->max_input)
        ruby_xfree_sized(tc->readbuf.ptr, tr->max_input);
    if ((int)sizeof(tc->writebuf.ary) < tr->max_output)
        ruby_xfree_sized(tc->writebuf.ptr, tr->max_output);
    SIZED_FREE(tc);
}
 
static size_t
rb_transcoding_memsize(rb_transcoding *tc)
{
    size_t size = sizeof(rb_transcoding);
    const rb_transcoder *tr = tc->transcoder;
 
    if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
        size += tr->state_size;
    }
    if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
        size += tr->max_input;
    }
    if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
        size += tr->max_output;
    }
    return size;
}
 
static rb_econv_t *
rb_econv_alloc(int n_hint)
{
    rb_econv_t *ec;
 
    if (n_hint <= 0)
        n_hint = 1;
 
    ec = ALLOC(rb_econv_t);
    ec->flags = 0;
    ec->source_encoding_name = NULL;
    ec->destination_encoding_name = NULL;
    ec->started = 0;
    ec->replacement_str = NULL;
    ec->replacement_len = 0;
    ec->replacement_bufsize = 0;
    ec->replacement_enc = NULL;
    ec->replacement_allocated = 0;
    ec->in_buf_start = NULL;
    ec->in_data_start = NULL;
    ec->in_data_end = NULL;
    ec->in_buf_end = NULL;
    ec->num_allocated = n_hint;
    ec->num_trans = 0;
    ec->elems = ALLOC_N(rb_econv_elem_t, ec->num_allocated);
    ec->num_finished = 0;
    ec->last_tc = NULL;
    ec->last_error.result = econv_source_buffer_empty;
    ec->last_error.error_tc = NULL;
    ec->last_error.source_encoding = NULL;
    ec->last_error.destination_encoding = NULL;
    ec->last_error.error_bytes_start = NULL;
    ec->last_error.error_bytes_len = 0;
    ec->last_error.readagain_len = 0;
    ec->source_encoding = NULL;
    ec->destination_encoding = NULL;
    return ec;
}
 
static int
rb_econv_add_transcoder_at(rb_econv_t *ec, const rb_transcoder *tr, int i)
{
    int n, j;
    int bufsize = 4096;
    unsigned char *p;
 
    if (ec->num_trans == ec->num_allocated) {
        n = ec->num_allocated * 2;
        SIZED_REALLOC_N(ec->elems, rb_econv_elem_t, n, ec->num_allocated);
        ec->num_allocated = n;
    }
 
    p = xmalloc(bufsize);
 
    MEMMOVE(ec->elems+i+1, ec->elems+i, rb_econv_elem_t, ec->num_trans-i);
 
    ec->elems[i].tc = rb_transcoding_open_by_transcoder(tr, 0);
    ec->elems[i].out_buf_start = p;
    ec->elems[i].out_buf_end = p + bufsize;
    ec->elems[i].out_data_start = p;
    ec->elems[i].out_data_end = p;
    ec->elems[i].last_result = econv_source_buffer_empty;
 
    ec->num_trans++;
 
    if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
        for (j = ec->num_trans-1; i <= j; j--) {
            rb_transcoding *tc = ec->elems[j].tc;
            const rb_transcoder *tr2 = tc->transcoder;
            if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
                ec->last_tc = tc;
                break;
            }
        }
 
    return 0;
}
 
static rb_econv_t *
rb_econv_open_by_transcoder_entries(int n, transcoder_entry_t **entries)
{
    rb_econv_t *ec;
    int i, ret;
 
    for (i = 0; i < n; i++) {
        const rb_transcoder *tr;
        tr = load_transcoder_entry(entries[i]);
        if (!tr)
            return NULL;
    }
 
    ec = rb_econv_alloc(n);
 
    for (i = 0; i < n; i++) {
        const rb_transcoder *tr = load_transcoder_entry(entries[i]);
        ret = rb_econv_add_transcoder_at(ec, tr, ec->num_trans);
        if (ret == -1) {
            rb_econv_close(ec);
            return NULL;
        }
    }
 
    return ec;
}
 
struct trans_open_t {
    transcoder_entry_t **entries;
};
 
static void
trans_open_i(const char *sname, const char *dname, int depth, void *arg)
{
    struct trans_open_t *toarg = arg;
 
    if (!toarg->entries) {
        toarg->entries = ALLOC_N(transcoder_entry_t *, depth + 1);
    }
    toarg->entries[depth] = get_transcoder_entry(sname, dname);
}
 
static rb_econv_t *
rb_econv_open0(const char *sname, const char *dname, int ecflags)
{
    transcoder_entry_t **entries = NULL;
    int num_trans;
    rb_econv_t *ec;
 
    // loads encodings if not loaded already
    if (*sname) rb_enc_find_index(sname);
    if (*dname) rb_enc_find_index(dname);
 
    if (*sname == '\0' && *dname == '\0') {
        num_trans = 0;
        entries = NULL;
        sname = dname = "";
    }
    else {
        struct trans_open_t toarg = {0};
        num_trans = transcode_search_path(sname, dname, trans_open_i, (void *)&toarg);
        entries = toarg.entries;
        if (num_trans < 0) {
            SIZED_FREE_N(entries, num_trans);
            return NULL;
        }
    }
 
    ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
    SIZED_FREE_N(entries, num_trans);
    if (!ec)
        return NULL;
 
    ec->flags = ecflags;
    ec->source_encoding_name = sname;
    ec->destination_encoding_name = dname;
 
    return ec;
}
 
#define MAX_ECFLAGS_DECORATORS 32
 
static int
decorator_names(int ecflags, const char **decorators_ret)
{
    int num_decorators;
 
    switch (ecflags & ECONV_NEWLINE_DECORATOR_MASK) {
      case ECONV_UNIVERSAL_NEWLINE_DECORATOR:
      case ECONV_CRLF_NEWLINE_DECORATOR:
      case ECONV_CR_NEWLINE_DECORATOR:
      case ECONV_LF_NEWLINE_DECORATOR:
      case 0:
        break;
      default:
        return -1;
    }
 
    if ((ecflags & ECONV_XML_TEXT_DECORATOR) &&
        (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR))
        return -1;
 
    num_decorators = 0;
 
    if (ecflags & ECONV_XML_TEXT_DECORATOR)
        decorators_ret[num_decorators++] = "xml_text_escape";
    if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR)
        decorators_ret[num_decorators++] = "xml_attr_content_escape";
    if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR)
        decorators_ret[num_decorators++] = "xml_attr_quote";
 
    if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR)
        decorators_ret[num_decorators++] = "crlf_newline";
    if (ecflags & ECONV_CR_NEWLINE_DECORATOR)
        decorators_ret[num_decorators++] = "cr_newline";
    if (ecflags & ECONV_LF_NEWLINE_DECORATOR)
        decorators_ret[num_decorators++] = "lf_newline";
    if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR)
        decorators_ret[num_decorators++] = "universal_newline";
 
    return num_decorators;
}
 
rb_econv_t *
rb_econv_open(const char *sname, const char *dname, int ecflags)
{
    rb_econv_t *ec;
    int num_decorators;
    const char *decorators[MAX_ECFLAGS_DECORATORS];
    int i;
 
    num_decorators = decorator_names(ecflags, decorators);
    if (num_decorators == -1)
        return NULL;
 
    ec = rb_econv_open0(sname, dname, ecflags & ECONV_ERROR_HANDLER_MASK);
    if (ec) {
        for (i = 0; i < num_decorators; i++) {
            if (rb_econv_decorate_at_last(ec, decorators[i]) == -1) {
                rb_econv_close(ec);
                ec = NULL;
                break;
            }
        }
    }
 
    if (ec) {
        ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
    }
    return ec; // can be NULL
}
 
static int
trans_sweep(rb_econv_t *ec,
    const unsigned char **input_ptr, const unsigned char *input_stop,
    unsigned char **output_ptr, unsigned char *output_stop,
    int flags,
    int start)
{
    int try;
    int i, f;
 
    const unsigned char **ipp, *is, *iold;
    unsigned char **opp, *os, *oold;
    rb_econv_result_t res;
 
    try = 1;
    while (try) {
        try = 0;
        for (i = start; i < ec->num_trans; i++) {
            rb_econv_elem_t *te = &ec->elems[i];
 
            if (i == 0) {
                ipp = input_ptr;
                is = input_stop;
            }
            else {
                rb_econv_elem_t *prev_te = &ec->elems[i-1];
                ipp = (const unsigned char **)&prev_te->out_data_start;
                is = prev_te->out_data_end;
            }
 
            if (i == ec->num_trans-1) {
                opp = output_ptr;
                os = output_stop;
            }
            else {
                if (te->out_buf_start != te->out_data_start) {
                    ssize_t len = te->out_data_end - te->out_data_start;
                    ssize_t off = te->out_data_start - te->out_buf_start;
                    MEMMOVE(te->out_buf_start, te->out_data_start, unsigned char, len);
                    te->out_data_start = te->out_buf_start;
                    te->out_data_end -= off;
                }
                opp = &te->out_data_end;
                os = te->out_buf_end;
            }
 
            f = flags;
            if (ec->num_finished != i)
                f |= ECONV_PARTIAL_INPUT;
            if (i == 0 && (flags & ECONV_AFTER_OUTPUT)) {
                start = 1;
                flags &= ~ECONV_AFTER_OUTPUT;
            }
            if (i != 0)
                f &= ~ECONV_AFTER_OUTPUT;
            iold = *ipp;
            oold = *opp;
            te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
            if (iold != *ipp || oold != *opp)
                try = 1;
 
            switch (res) {
              case econv_invalid_byte_sequence:
              case econv_incomplete_input:
              case econv_undefined_conversion:
              case econv_after_output:
                return i;
 
              case econv_destination_buffer_full:
              case econv_source_buffer_empty:
                break;
 
              case econv_finished:
                ec->num_finished = i+1;
                break;
            }
        }
    }
    return -1;
}
 
static rb_econv_result_t
rb_trans_conv(rb_econv_t *ec,
    const unsigned char **input_ptr, const unsigned char *input_stop,
    unsigned char **output_ptr, unsigned char *output_stop,
    int flags,
    int *result_position_ptr)
{
    int i;
    int needreport_index;
    int sweep_start;
 
    unsigned char empty_buf;
    unsigned char *empty_ptr = &empty_buf;
 
    if (!input_ptr) {
        input_ptr = (const unsigned char **)&empty_ptr;
        input_stop = empty_ptr;
    }
 
    if (!output_ptr) {
        output_ptr = &empty_ptr;
        output_stop = empty_ptr;
    }
 
    if (ec->elems[0].last_result == econv_after_output)
        ec->elems[0].last_result = econv_source_buffer_empty;
 
    for (i = ec->num_trans-1; 0 <= i; i--) {
        switch (ec->elems[i].last_result) {
          case econv_invalid_byte_sequence:
          case econv_incomplete_input:
          case econv_undefined_conversion:
          case econv_after_output:
          case econv_finished:
            sweep_start = i+1;
            goto found_needreport;
 
          case econv_destination_buffer_full:
          case econv_source_buffer_empty:
            break;
 
          default:
            rb_bug("unexpected transcode last result");
        }
    }
 
    /* /^[sd]+$/ is confirmed.  but actually /^s*d*$/. */
 
    if (ec->elems[ec->num_trans-1].last_result == econv_destination_buffer_full &&
        (flags & ECONV_AFTER_OUTPUT)) {
        rb_econv_result_t res;
 
        res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
                (flags & ~ECONV_AFTER_OUTPUT)|ECONV_PARTIAL_INPUT,
                result_position_ptr);
 
        if (res == econv_source_buffer_empty)
            return econv_after_output;
        return res;
    }
 
    sweep_start = 0;
 
  found_needreport:
 
    do {
        needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
        sweep_start = needreport_index + 1;
    } while (needreport_index != -1 && needreport_index != ec->num_trans-1);
 
    for (i = ec->num_trans-1; 0 <= i; i--) {
        if (ec->elems[i].last_result != econv_source_buffer_empty) {
            rb_econv_result_t res = ec->elems[i].last_result;
            if (res == econv_invalid_byte_sequence ||
                res == econv_incomplete_input ||
                res == econv_undefined_conversion ||
                res == econv_after_output) {
                ec->elems[i].last_result = econv_source_buffer_empty;
            }
            if (result_position_ptr)
                *result_position_ptr = i;
            return res;
        }
    }
    if (result_position_ptr)
        *result_position_ptr = -1;
    return econv_source_buffer_empty;
}
 
static rb_econv_result_t
rb_econv_convert0(rb_econv_t *ec,
    const unsigned char **input_ptr, const unsigned char *input_stop,
    unsigned char **output_ptr, unsigned char *output_stop,
    int flags)
{
    rb_econv_result_t res;
    int result_position;
    int has_output = 0;
 
    memset(&ec->last_error, 0, sizeof(ec->last_error));
 
    if (ec->num_trans == 0) {
        size_t len;
        if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
            if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
                len = output_stop - *output_ptr;
                memcpy(*output_ptr, ec->in_data_start, len);
                *output_ptr = output_stop;
                ec->in_data_start += len;
                res = econv_destination_buffer_full;
                goto gotresult;
            }
            len = ec->in_data_end - ec->in_data_start;
            memcpy(*output_ptr, ec->in_data_start, len);
            *output_ptr += len;
            ec->in_data_start = ec->in_data_end = ec->in_buf_start;
            if (flags & ECONV_AFTER_OUTPUT) {
                res = econv_after_output;
                goto gotresult;
            }
        }
        if (output_stop - *output_ptr < input_stop - *input_ptr) {
            len = output_stop - *output_ptr;
        }
        else {
            len = input_stop - *input_ptr;
        }
        if (0 < len && (flags & ECONV_AFTER_OUTPUT)) {
            *(*output_ptr)++ = *(*input_ptr)++;
            res = econv_after_output;
            goto gotresult;
        }
        memcpy(*output_ptr, *input_ptr, len);
        *output_ptr += len;
        *input_ptr += len;
        if (*input_ptr != input_stop)
            res = econv_destination_buffer_full;
        else if (flags & ECONV_PARTIAL_INPUT)
            res = econv_source_buffer_empty;
        else
            res = econv_finished;
        goto gotresult;
    }
 
    if (ec->elems[ec->num_trans-1].out_data_start) {
        unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
        unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
        if (data_start != data_end) {
            size_t len;
            if (output_stop - *output_ptr < data_end - data_start) {
                len = output_stop - *output_ptr;
                memcpy(*output_ptr, data_start, len);
                *output_ptr = output_stop;
                ec->elems[ec->num_trans-1].out_data_start += len;
                res = econv_destination_buffer_full;
                goto gotresult;
            }
            len = data_end - data_start;
            memcpy(*output_ptr, data_start, len);
            *output_ptr += len;
            ec->elems[ec->num_trans-1].out_data_start =
                ec->elems[ec->num_trans-1].out_data_end =
                ec->elems[ec->num_trans-1].out_buf_start;
            has_output = 1;
        }
    }
 
    if (ec->in_buf_start &&
        ec->in_data_start != ec->in_data_end) {
        res = rb_trans_conv(ec, (const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
                (flags&~ECONV_AFTER_OUTPUT)|ECONV_PARTIAL_INPUT, &result_position);
        if (res != econv_source_buffer_empty)
            goto gotresult;
    }
 
    if (has_output &&
        (flags & ECONV_AFTER_OUTPUT) &&
        *input_ptr != input_stop) {
        input_stop = *input_ptr;
        res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
        if (res == econv_source_buffer_empty)
            res = econv_after_output;
    }
    else if ((flags & ECONV_AFTER_OUTPUT) ||
        ec->num_trans == 1) {
        res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
    }
    else {
        flags |= ECONV_AFTER_OUTPUT;
        do {
            res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
        } while (res == econv_after_output);
    }
 
  gotresult:
    ec->last_error.result = res;
    if (res == econv_invalid_byte_sequence ||
        res == econv_incomplete_input ||
        res == econv_undefined_conversion) {
        rb_transcoding *error_tc = ec->elems[result_position].tc;
        ec->last_error.error_tc = error_tc;
        ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
        ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
        ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
        ec->last_error.error_bytes_len = error_tc->recognized_len;
        ec->last_error.readagain_len = error_tc->readagain_len;
    }
 
    return res;
}
 
static int output_replacement_character(rb_econv_t *ec);
 
static int
output_hex_charref(rb_econv_t *ec)
{
    int ret;
    unsigned char utfbuf[1024];
    const unsigned char *utf;
    size_t utf_len, utf_bufsize;
    int utf_allocated = 0;
    char charef_buf[16];
    const unsigned char *p;
 
    if (encoding_equal(ec->last_error.source_encoding, "UTF-32BE")) {
        utf = ec->last_error.error_bytes_start;
        utf_len = ec->last_error.error_bytes_len;
    }
    else {
        utf = allocate_converted_string(ec->last_error.source_encoding, "UTF-32BE",
                ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
                utfbuf, sizeof(utfbuf),
                &utf_len, &utf_bufsize);
        if (!utf)
            return -1;
        if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
            utf_allocated = 1;
    }
 
    if (utf_len % 4 != 0)
        goto fail;
 
    p = utf;
    while (4 <= utf_len) {
        unsigned int u = 0;
        u += p[0] << 24;
        u += p[1] << 16;
        u += p[2] << 8;
        u += p[3];
        snprintf(charef_buf, sizeof(charef_buf), "&#x%X;", u);
 
        ret = rb_econv_insert_output(ec, (unsigned char *)charef_buf, strlen(charef_buf), "US-ASCII");
        if (ret == -1)
            goto fail;
 
        p += 4;
        utf_len -= 4;
    }
 
    if (utf_allocated)
        ruby_xfree_sized((void *)utf, utf_bufsize);
    return 0;
 
  fail:
    if (utf_allocated)
        ruby_xfree_sized((void *)utf, utf_bufsize);
    return -1;
}
 
rb_econv_result_t
rb_econv_convert(rb_econv_t *ec,
    const unsigned char **input_ptr, const unsigned char *input_stop,
    unsigned char **output_ptr, unsigned char *output_stop,
    int flags)
{
    rb_econv_result_t ret;
 
    unsigned char empty_buf;
    unsigned char *empty_ptr = &empty_buf;
 
    ec->started = 1;
 
    if (!input_ptr) {
        input_ptr = (const unsigned char **)&empty_ptr;
        input_stop = empty_ptr;
    }
 
    if (!output_ptr) {
        output_ptr = &empty_ptr;
        output_stop = empty_ptr;
    }
 
  resume:
    ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
 
    if (ret == econv_invalid_byte_sequence ||
        ret == econv_incomplete_input) {
        /* deal with invalid byte sequence */
        /* todo: add more alternative behaviors */
        switch (ec->flags & ECONV_INVALID_MASK) {
          case ECONV_INVALID_REPLACE:
            if (output_replacement_character(ec) == 0)
                goto resume;
        }
    }
 
    if (ret == econv_undefined_conversion) {
        /* valid character in source encoding
         * but no related character(s) in destination encoding */
        /* todo: add more alternative behaviors */
        switch (ec->flags & ECONV_UNDEF_MASK) {
          case ECONV_UNDEF_REPLACE:
            if (output_replacement_character(ec) == 0)
                goto resume;
            break;
 
          case ECONV_UNDEF_HEX_CHARREF:
            if (output_hex_charref(ec) == 0)
                goto resume;
            break;
        }
    }
 
    return ret;
}
 
const char *
rb_econv_encoding_to_insert_output(rb_econv_t *ec)
{
    rb_transcoding *tc = ec->last_tc;
    const rb_transcoder *tr;
 
    if (tc == NULL)
        return "";
 
    tr = tc->transcoder;
 
    if (tr->asciicompat_type == asciicompat_encoder)
        return tr->src_encoding;
    return tr->dst_encoding;
}
 
static unsigned char *
allocate_converted_string(const char *sname, const char *dname,
        const unsigned char *str, size_t len,
        unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
        size_t *dst_len_ptr, size_t *dst_bufsize_ptr)
{
    unsigned char *dst_str;
    size_t dst_len;
    size_t dst_bufsize;
 
    rb_econv_t *ec;
    rb_econv_result_t res;
 
    const unsigned char *sp;
    unsigned char *dp;
 
    if (caller_dst_buf)
        dst_bufsize = caller_dst_bufsize;
    else if (len == 0)
        dst_bufsize = 1;
    else
        dst_bufsize = len;
 
    ec = rb_econv_open(sname, dname, 0);
    if (ec == NULL)
        return NULL;
    if (caller_dst_buf)
        dst_str = caller_dst_buf;
    else
        dst_str = xmalloc(dst_bufsize);
    dst_len = 0;
    sp = str;
    dp = dst_str+dst_len;
    res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
    dst_len = dp - dst_str;
    while (res == econv_destination_buffer_full) {
        if (SIZE_MAX/2 < dst_bufsize) {
            goto fail;
        }
        dst_bufsize *= 2;
        if (dst_str == caller_dst_buf) {
            unsigned char *tmp;
            tmp = xmalloc(dst_bufsize);
            memcpy(tmp, dst_str, dst_bufsize/2);
            dst_str = tmp;
        }
        else {
            dst_str = ruby_xrealloc_sized(dst_str, dst_bufsize, dst_bufsize / 2);
        }
        dp = dst_str+dst_len;
        res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
        dst_len = dp - dst_str;
    }
    if (res != econv_finished) {
        goto fail;
    }
    rb_econv_close(ec);
    *dst_len_ptr = dst_len;
    *dst_bufsize_ptr = dst_bufsize;
    return dst_str;
 
  fail:
    if (dst_str != caller_dst_buf)
        ruby_xfree_sized(dst_str, dst_bufsize);
    rb_econv_close(ec);
    return NULL;
}
 
/* result: 0:success -1:failure */
int
rb_econv_insert_output(rb_econv_t *ec,
    const unsigned char *str, size_t len, const char *str_encoding)
{
    const char *insert_encoding = rb_econv_encoding_to_insert_output(ec);
    unsigned char insert_buf[4096];
    const unsigned char *insert_str = NULL;
    size_t insert_len, insert_bufsize;
 
    int last_trans_index;
    rb_transcoding *tc;
 
    unsigned char **buf_start_p;
    unsigned char **data_start_p;
    unsigned char **data_end_p;
    unsigned char **buf_end_p;
 
    size_t need;
 
    ec->started = 1;
 
    if (len == 0)
        return 0;
 
    if (encoding_equal(insert_encoding, str_encoding)) {
        insert_str = str;
        insert_len = len;
    }
    else {
        insert_str = allocate_converted_string(str_encoding, insert_encoding,
                str, len, insert_buf, sizeof(insert_buf), &insert_len, &insert_bufsize);
        if (insert_str == NULL)
            return -1;
    }
 
    need = insert_len;
 
    last_trans_index = ec->num_trans-1;
    if (ec->num_trans == 0) {
        tc = NULL;
        buf_start_p = &ec->in_buf_start;
        data_start_p = &ec->in_data_start;
        data_end_p = &ec->in_data_end;
        buf_end_p = &ec->in_buf_end;
    }
    else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
        tc = ec->elems[last_trans_index].tc;
        need += tc->readagain_len;
        if (need < insert_len)
            goto fail;
        if (last_trans_index == 0) {
            buf_start_p = &ec->in_buf_start;
            data_start_p = &ec->in_data_start;
            data_end_p = &ec->in_data_end;
            buf_end_p = &ec->in_buf_end;
        }
        else {
            rb_econv_elem_t *ee = &ec->elems[last_trans_index-1];
            buf_start_p = &ee->out_buf_start;
            data_start_p = &ee->out_data_start;
            data_end_p = &ee->out_data_end;
            buf_end_p = &ee->out_buf_end;
        }
    }
    else {
        rb_econv_elem_t *ee = &ec->elems[last_trans_index];
        buf_start_p = &ee->out_buf_start;
        data_start_p = &ee->out_data_start;
        data_end_p = &ee->out_data_end;
        buf_end_p = &ee->out_buf_end;
        tc = ec->elems[last_trans_index].tc;
    }
 
    if (*buf_start_p == NULL) {
        unsigned char *buf = xmalloc(need);
        *buf_start_p = buf;
        *data_start_p = buf;
        *data_end_p = buf;
        *buf_end_p = buf+need;
    }
    else if ((size_t)(*buf_end_p - *data_end_p) < need) {
        MEMMOVE(*buf_start_p, *data_start_p, unsigned char, *data_end_p - *data_start_p);
        *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
        *data_start_p = *buf_start_p;
        if ((size_t)(*buf_end_p - *data_end_p) < need) {
            unsigned char *buf;
            size_t s = (*data_end_p - *buf_start_p) + need;
            if (s < need)
                goto fail;
            buf = ruby_xrealloc_sized(*buf_start_p, s, *buf_end_p - *buf_start_p);
            *data_start_p = buf;
            *data_end_p = buf + (*data_end_p - *buf_start_p);
            *buf_start_p = buf;
            *buf_end_p = buf + s;
        }
    }
 
    memcpy(*data_end_p, insert_str, insert_len);
    *data_end_p += insert_len;
    if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
        memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
        *data_end_p += tc->readagain_len;
        tc->readagain_len = 0;
    }
 
    if (insert_str != str && insert_str != insert_buf)
        ruby_xfree_sized((void *)insert_str, insert_bufsize);
    return 0;
 
  fail:
    if (insert_str != str && insert_str != insert_buf)
        ruby_xfree_sized((void *)insert_str, insert_bufsize);
    return -1;
}
 
void
rb_econv_close(rb_econv_t *ec)
{
    int i;
 
    if (ec->replacement_allocated) {
        SIZED_FREE_N((char *)ec->replacement_str, ec->replacement_bufsize);
    }
    for (i = 0; i < ec->num_trans; i++) {
        rb_transcoding_close(ec->elems[i].tc);
        ruby_xfree_sized(ec->elems[i].out_buf_start, ec->elems[i].out_buf_end - ec->elems[i].out_buf_start);
    }
    SIZED_FREE_N(ec->in_buf_start, ec->in_buf_end - ec->in_buf_start);
    SIZED_FREE_N(ec->elems, ec->num_allocated);
    SIZED_FREE(ec);
}
 
size_t
rb_econv_memsize(rb_econv_t *ec)
{
    size_t size = sizeof(rb_econv_t);
    int i;
 
    if (ec->replacement_allocated) {
        size += ec->replacement_len;
    }
    for (i = 0; i < ec->num_trans; i++) {
        size += rb_transcoding_memsize(ec->elems[i].tc);
 
        if (ec->elems[i].out_buf_start) {
            size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
        }
    }
    size += ec->in_buf_end - ec->in_buf_start;
    size += sizeof(rb_econv_elem_t) * ec->num_allocated;
 
    return size;
}
 
int
rb_econv_putbackable(rb_econv_t *ec)
{
    if (ec->num_trans == 0)
        return 0;
#if SIZEOF_SIZE_T > SIZEOF_INT
    if (ec->elems[0].tc->readagain_len > INT_MAX) return INT_MAX;
#endif
    return (int)ec->elems[0].tc->readagain_len;
}
 
void
rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
{
    rb_transcoding *tc;
    if (ec->num_trans == 0 || n == 0)
        return;
    tc = ec->elems[0].tc;
    memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
    tc->readagain_len -= n;
}
 
struct asciicompat_encoding_t {
    const char *ascii_compat_name;
    const char *ascii_incompat_name;
};
 
static int
asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
{
    struct asciicompat_encoding_t *data = (struct asciicompat_encoding_t *)arg;
    transcoder_entry_t *entry = (transcoder_entry_t *)val;
    const rb_transcoder *tr;
 
    if (DECORATOR_P(entry->sname, entry->dname))
        return ST_CONTINUE;
    tr = load_transcoder_entry(entry);
    if (tr && tr->asciicompat_type == asciicompat_decoder) {
        data->ascii_compat_name = tr->dst_encoding;
        return ST_STOP;
    }
    return ST_CONTINUE;
}
 
const char *
rb_econv_asciicompat_encoding(const char *ascii_incompat_name)
{
    st_data_t v;
    st_table *table2;
    struct asciicompat_encoding_t data = {0};
 
    unsigned int lev;
    RB_VM_LOCK_ENTER_LEV(&lev);
    {
        if (st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v)) {
            table2 = (st_table *)v;
            /*
            * Assumption:
            * There is at most one transcoder for
            * converting from ASCII incompatible encoding.
            *
            * For ISO-2022-JP, there is ISO-2022-JP -> stateless-ISO-2022-JP and no others.
            */
            if (table2->num_entries == 1) {
                data.ascii_incompat_name = ascii_incompat_name;
                data.ascii_compat_name = NULL;
                if (rb_multi_ractor_p()) {
                    /*
                     * We need to unlock in case `load_transcoder_entry` actually loads the encoding
                     * and table2 could be inserted into when we unlock.
                     */
                    st_table *dup_table2 = st_copy(table2);
                    RB_VM_LOCK_LEAVE_LEV(&lev);
                    st_foreach(dup_table2, asciicompat_encoding_i, (st_data_t)&data);
                    st_free_table(dup_table2);
                    RB_VM_LOCK_ENTER_LEV(&lev);
                }
                else {
                    st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
                }
            }
 
        }
    }
    RB_VM_LOCK_LEAVE_LEV(&lev);
 
    return data.ascii_compat_name; // can be NULL
}
 
/*
 * Append `len` bytes pointed by `ss` to `dst` with converting with `ec`.
 *
 * If the result of the conversion is not compatible with the encoding of
 * `dst`, `dst` may not be valid encoding.
 */
VALUE
rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags)
{
    unsigned const char *sp, *se;
    unsigned char *ds, *dp, *de;
    rb_econv_result_t res;
    int max_output;
    enum ruby_coderange_type coderange;
    rb_encoding *dst_enc = ec->destination_encoding;
 
    if (NIL_P(dst)) {
        dst = rb_str_buf_new(len);
        if (dst_enc) {
            rb_enc_associate(dst, dst_enc);
        }
        coderange = ENC_CODERANGE_7BIT; // scan from the start
    }
    else {
        dst_enc = rb_enc_get(dst);
        coderange = rb_enc_str_coderange(dst);
    }
 
    if (ec->last_tc)
        max_output = ec->last_tc->transcoder->max_output;
    else
        max_output = 1;
 
    do {
        int cr;
        long dlen = RSTRING_LEN(dst);
        if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) {
            unsigned long new_capa = (unsigned long)dlen + len + max_output;
            if (LONG_MAX < new_capa)
                rb_raise(rb_eArgError, "too long string");
            rb_str_modify_expand(dst, new_capa - dlen);
        }
        sp = (const unsigned char *)ss;
        se = sp + len;
        ds = (unsigned char *)RSTRING_PTR(dst);
        de = ds + rb_str_capacity(dst);
        dp = ds += dlen;
        res = rb_econv_convert(ec, &sp, se, &dp, de, flags);
        switch (coderange) {
          case ENC_CODERANGE_7BIT:
          case ENC_CODERANGE_VALID:
            cr = (int)coderange;
            rb_str_coderange_scan_restartable((char *)ds, (char *)dp, dst_enc, &cr);
            coderange = cr;
            ENC_CODERANGE_SET(dst, coderange);
            break;
          case ENC_CODERANGE_UNKNOWN:
          case ENC_CODERANGE_BROKEN:
            break;
        }
        len -= (const char *)sp - ss;
        ss  = (const char *)sp;
        rb_str_set_len(dst, dlen + (dp - ds));
        rb_econv_check_error(ec);
    } while (res == econv_destination_buffer_full);
 
    return dst;
}
 
VALUE
rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags)
{
    src = rb_str_new_frozen(src);
    dst = rb_econv_append(ec, RSTRING_PTR(src) + off, len, dst, flags);
    RB_GC_GUARD(src);
    return dst;
}
 
VALUE
rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
{
    return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), dst, flags);
}
 
VALUE
rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
{
    return rb_econv_substr_append(ec, src, byteoff, bytesize, Qnil, flags);
}
 
VALUE
rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
{
    return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), Qnil, flags);
}
 
static int
rb_econv_add_converter(rb_econv_t *ec, const char *sname, const char *dname, int n)
{
    transcoder_entry_t *entry;
    const rb_transcoder *tr = NULL;
 
    if (ec->started != 0)
        return -1;
 
    entry = get_transcoder_entry(sname, dname);
    if (entry) {
        tr = load_transcoder_entry(entry);
    }
 
    return tr ? rb_econv_add_transcoder_at(ec, tr, n) : -1;
}
 
static int
rb_econv_decorate_at(rb_econv_t *ec, const char *decorator_name, int n)
{
    return rb_econv_add_converter(ec, "", decorator_name, n);
}
 
int
rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
{
    const rb_transcoder *tr;
 
    if (ec->num_trans == 0)
        return rb_econv_decorate_at(ec, decorator_name, 0);
 
    tr = ec->elems[0].tc->transcoder;
 
    if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
        tr->asciicompat_type == asciicompat_decoder)
        return rb_econv_decorate_at(ec, decorator_name, 1);
 
    return rb_econv_decorate_at(ec, decorator_name, 0);
}
 
int
rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
{
    const rb_transcoder *tr;
 
    if (ec->num_trans == 0)
        return rb_econv_decorate_at(ec, decorator_name, 0);
 
    tr = ec->elems[ec->num_trans-1].tc->transcoder;
 
    if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
        tr->asciicompat_type == asciicompat_encoder)
        return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
 
    return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
}
 
void
rb_econv_binmode(rb_econv_t *ec)
{
    const char *dname = 0;
 
    switch (ec->flags & ECONV_NEWLINE_DECORATOR_MASK) {
      case ECONV_UNIVERSAL_NEWLINE_DECORATOR:
        dname = "universal_newline";
        break;
      case ECONV_CRLF_NEWLINE_DECORATOR:
        dname = "crlf_newline";
        break;
      case ECONV_CR_NEWLINE_DECORATOR:
        dname = "cr_newline";
        break;
      case ECONV_LF_NEWLINE_DECORATOR:
        dname = "lf_newline";
        break;
    }
 
    if (dname) {
        const rb_transcoder *transcoder = get_transcoder_entry("", dname)->transcoder;
        int num_trans = ec->num_trans;
        int i, j = 0;
 
        for (i=0; i < num_trans; i++) {
            if (transcoder == ec->elems[i].tc->transcoder) {
                rb_transcoding_close(ec->elems[i].tc);
                ruby_xfree_sized(ec->elems[i].out_buf_start, ec->elems[i].out_buf_end - ec->elems[i].out_buf_start);
                ec->num_trans--;
            }
            else
                ec->elems[j++] = ec->elems[i];
        }
    }
 
    ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
}
 
static VALUE
econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg)
{
    int has_description = 0;
 
    if (NIL_P(mesg))
        mesg = rb_str_new(NULL, 0);
 
    if (*sname != '\0' || *dname != '\0') {
        if (*sname == '\0')
            rb_str_cat2(mesg, dname);
        else if (*dname == '\0')
            rb_str_cat2(mesg, sname);
        else
            rb_str_catf(mesg, "%s to %s", sname, dname);
        has_description = 1;
    }
 
    if (ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
                   ECONV_XML_TEXT_DECORATOR|
                   ECONV_XML_ATTR_CONTENT_DECORATOR|
                   ECONV_XML_ATTR_QUOTE_DECORATOR)) {
        const char *pre = "";
        if (has_description)
            rb_str_cat2(mesg, " with ");
        if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR)  {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "universal_newline");
        }
        if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "crlf_newline");
        }
        if (ecflags & ECONV_CR_NEWLINE_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "cr_newline");
        }
        if (ecflags & ECONV_LF_NEWLINE_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "lf_newline");
        }
        if (ecflags & ECONV_XML_TEXT_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "xml_text");
        }
        if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "xml_attr_content");
        }
        if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR) {
            rb_str_cat2(mesg, pre); pre = ",";
            rb_str_cat2(mesg, "xml_attr_quote");
        }
        has_description = 1;
    }
    if (!has_description) {
        rb_str_cat2(mesg, "no-conversion");
    }
 
    return mesg;
}
 
VALUE
rb_econv_open_exc(const char *sname, const char *dname, int ecflags)
{
    VALUE mesg, exc;
    mesg = rb_str_new_cstr("code converter not found (");
    econv_description(sname, dname, ecflags, mesg);
    rb_str_cat2(mesg, ")");
    exc = rb_exc_new3(rb_eConverterNotFoundError, mesg);
    return exc;
}
 
static VALUE
make_econv_exception(rb_econv_t *ec)
{
    VALUE mesg, exc;
    if (ec->last_error.result == econv_invalid_byte_sequence ||
        ec->last_error.result == econv_incomplete_input) {
        const char *err = (const char *)ec->last_error.error_bytes_start;
        size_t error_len = ec->last_error.error_bytes_len;
        VALUE bytes = rb_str_new(err, error_len);
        VALUE dumped = rb_str_dump(bytes);
        size_t readagain_len = ec->last_error.readagain_len;
        VALUE bytes2 = Qnil;
        VALUE dumped2;
        if (ec->last_error.result == econv_incomplete_input) {
            mesg = rb_sprintf("incomplete %s on %s",
                    StringValueCStr(dumped),
                    ec->last_error.source_encoding);
        }
        else if (readagain_len) {
            bytes2 = rb_str_new(err+error_len, readagain_len);
            dumped2 = rb_str_dump(bytes2);
            mesg = rb_sprintf("%s followed by %s on %s",
                    StringValueCStr(dumped),
                    StringValueCStr(dumped2),
                    ec->last_error.source_encoding);
        }
        else {
            mesg = rb_sprintf("%s on %s",
                    StringValueCStr(dumped),
                    ec->last_error.source_encoding);
        }
 
        exc = rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
        rb_ivar_set(exc, id_error_bytes, bytes);
        rb_ivar_set(exc, id_readagain_bytes, bytes2);
        rb_ivar_set(exc, id_incomplete_input, RBOOL(ec->last_error.result == econv_incomplete_input));
        goto set_encs;
    }
    if (ec->last_error.result == econv_undefined_conversion) {
        VALUE bytes = rb_str_new((const char *)ec->last_error.error_bytes_start,
                                 ec->last_error.error_bytes_len);
        VALUE dumped = Qnil;
        int idx;
        if (strcmp(ec->last_error.source_encoding, "UTF-8") == 0) {
            rb_encoding *utf8 = rb_utf8_encoding();
            const char *start, *end;
            int n;
            start = (const char *)ec->last_error.error_bytes_start;
            end = start + ec->last_error.error_bytes_len;
            n = rb_enc_precise_mbclen(start, end, utf8);
            if (MBCLEN_CHARFOUND_P(n) &&
                (size_t)MBCLEN_CHARFOUND_LEN(n) == ec->last_error.error_bytes_len) {
                unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
                dumped = rb_sprintf("U+%04X", cc);
            }
        }
        if (NIL_P(dumped))
            dumped = rb_str_dump(bytes);
        if (strcmp(ec->last_error.source_encoding,
                   ec->source_encoding_name) == 0 &&
            strcmp(ec->last_error.destination_encoding,
                   ec->destination_encoding_name) == 0) {
            mesg = rb_sprintf("%s from %s to %s",
                    StringValueCStr(dumped),
                    ec->last_error.source_encoding,
                    ec->last_error.destination_encoding);
        }
        else {
            int i;
            mesg = rb_sprintf("%s to %s in conversion from %s",
                    StringValueCStr(dumped),
                    ec->last_error.destination_encoding,
                    ec->source_encoding_name);
            for (i = 0; i < ec->num_trans; i++) {
                const rb_transcoder *tr = ec->elems[i].tc->transcoder;
                if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
                    rb_str_catf(mesg, " to %s",
                                ec->elems[i].tc->transcoder->dst_encoding);
            }
        }
        exc = rb_exc_new3(rb_eUndefinedConversionError, mesg);
        idx = rb_enc_find_index(ec->last_error.source_encoding);
        if (0 <= idx)
            rb_enc_associate_index(bytes, idx);
        rb_ivar_set(exc, id_error_char, bytes);
        goto set_encs;
    }
    return Qnil;
 
  set_encs:
    rb_ivar_set(exc, id_source_encoding_name, rb_str_new2(ec->last_error.source_encoding));
    rb_ivar_set(exc, id_destination_encoding_name, rb_str_new2(ec->last_error.destination_encoding));
    int idx = rb_enc_find_index(ec->last_error.source_encoding);
    if (0 <= idx)
        rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
    idx = rb_enc_find_index(ec->last_error.destination_encoding);
    if (0 <= idx)
        rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
    return exc;
}
 
static void
more_output_buffer(
        VALUE destination,
        unsigned char *(*resize_destination)(VALUE, size_t, size_t),
        int max_output,
        unsigned char **out_start_ptr,
        unsigned char **out_pos,
        unsigned char **out_stop_ptr)
{
    size_t len = (*out_pos - *out_start_ptr);
    size_t new_len = (len + max_output) * 2;
    *out_start_ptr = resize_destination(destination, len, new_len);
    *out_pos = *out_start_ptr + len;
    *out_stop_ptr = *out_start_ptr + new_len;
}
 
static int
make_replacement(rb_econv_t *ec)
{
    rb_transcoding *tc;
    const rb_transcoder *tr;
    const unsigned char *replacement;
    const char *repl_enc;
    const char *ins_enc;
    size_t len;
 
    if (ec->replacement_str)
        return 0;
 
    ins_enc = rb_econv_encoding_to_insert_output(ec);
 
    tc = ec->last_tc;
    if (*ins_enc) {
        tr = tc->transcoder;
        rb_enc_find(tr->dst_encoding);
        replacement = (const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc);
    }
    else {
        replacement = (unsigned char *)"?";
        len = 1;
        repl_enc = "";
    }
 
    ec->replacement_str = replacement;
    ec->replacement_len = len;
    ec->replacement_bufsize = len;
    ec->replacement_enc = repl_enc;
    ec->replacement_allocated = 0;
    return 0;
}
 
int
rb_econv_set_replacement(rb_econv_t *ec,
    const unsigned char *str, size_t len, const char *encname)
{
    unsigned char *str2;
    size_t len2, buf_size2;
    const char *encname2;
 
    encname2 = rb_econv_encoding_to_insert_output(ec);
 
    if (!*encname2 || encoding_equal(encname, encname2)) {
        str2 = xmalloc(len);
        MEMCPY(str2, str, unsigned char, len); /* xxx: str may be invalid */
        buf_size2 = len2 = len;
        encname2 = encname;
    }
    else {
        str2 = allocate_converted_string(encname, encname2, str, len, NULL, 0, &len2, &buf_size2);
        if (!str2)
            return -1;
    }
 
    if (ec->replacement_allocated) {
        SIZED_FREE_N((char *)ec->replacement_str, ec->replacement_bufsize);
    }
    ec->replacement_allocated = 1;
    ec->replacement_str = str2;
    ec->replacement_len = len2;
    ec->replacement_bufsize = buf_size2;
    ec->replacement_enc = encname2;
    return 0;
}
 
static int
output_replacement_character(rb_econv_t *ec)
{
    int ret;
 
    if (make_replacement(ec) == -1)
        return -1;
 
    ret = rb_econv_insert_output(ec, ec->replacement_str, ec->replacement_len, ec->replacement_enc);
    if (ret == -1)
        return -1;
 
    return 0;
}
 
#if 1
#define hash_fallback rb_hash_aref
 
static VALUE
proc_fallback(VALUE fallback, VALUE c)
{
    return rb_proc_call(fallback, rb_ary_new4(1, &c));
}
 
static VALUE
method_fallback(VALUE fallback, VALUE c)
{
    return rb_method_call(1, &c, fallback);
}
 
static VALUE
aref_fallback(VALUE fallback, VALUE c)
{
    return rb_funcallv_public(fallback, idAREF, 1, &c);
}
 
struct transcode_loop_fallback_args {
    VALUE (*fallback_func)(VALUE, VALUE);
    VALUE fallback;
    VALUE rep;
};
 
static VALUE
transcode_loop_fallback_try(VALUE a)
{
    struct transcode_loop_fallback_args *args = (struct transcode_loop_fallback_args *)a;
 
    VALUE ret = args->fallback_func(args->fallback, args->rep);
 
    if (!UNDEF_P(ret) && !NIL_P(ret)) {
        StringValue(ret);
    }
 
    return ret;
}
 
static void
transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
               const unsigned char *in_stop, unsigned char *out_stop,
               VALUE destination,
               unsigned char *(*resize_destination)(VALUE, size_t, size_t),
               const char *src_encoding,
               const char *dst_encoding,
               int ecflags,
               VALUE ecopts)
{
    rb_econv_t *ec;
    rb_transcoding *last_tc;
    rb_econv_result_t ret;
    unsigned char *out_start = *out_pos;
    int max_output;
    VALUE exc;
    VALUE fallback = Qnil;
    VALUE (*fallback_func)(VALUE, VALUE) = 0;
 
    ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
    if (!ec)
        rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
 
    if (!NIL_P(ecopts) && RB_TYPE_P(ecopts, T_HASH)) {
        fallback = rb_hash_aref(ecopts, sym_fallback);
        if (RB_TYPE_P(fallback, T_HASH)) {
            fallback_func = hash_fallback;
        }
        else if (rb_obj_is_proc(fallback)) {
            fallback_func = proc_fallback;
        }
        else if (rb_obj_is_method(fallback)) {
            fallback_func = method_fallback;
        }
        else {
            fallback_func = aref_fallback;
        }
    }
    last_tc = ec->last_tc;
    max_output = last_tc ? last_tc->transcoder->max_output : 1;
 
  resume:
    ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0);
 
    if (!NIL_P(fallback) && ret == econv_undefined_conversion) {
        VALUE rep = rb_enc_str_new(
                (const char *)ec->last_error.error_bytes_start,
                ec->last_error.error_bytes_len,
                rb_enc_find(ec->last_error.source_encoding));
 
 
        struct transcode_loop_fallback_args args = {
            .fallback_func = fallback_func,
            .fallback = fallback,
            .rep = rep,
        };
 
        int state;
        rep = rb_protect(transcode_loop_fallback_try, (VALUE)&args, &state);
        if (state) {
            rb_econv_close(ec);
            rb_jump_tag(state);
        }
 
        if (!UNDEF_P(rep) && !NIL_P(rep)) {
            ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep),
                    RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
            RB_GC_GUARD(rep); // insert_output may GC while reading rep's bytes
            if ((int)ret == -1) {
                rb_econv_close(ec);
                rb_raise(rb_eArgError, "too big fallback string");
            }
            goto resume;
        }
    }
 
    if (ret == econv_invalid_byte_sequence ||
        ret == econv_incomplete_input ||
        ret == econv_undefined_conversion) {
        exc = make_econv_exception(ec);
        rb_econv_close(ec);
        rb_exc_raise(exc);
    }
 
    if (ret == econv_destination_buffer_full) {
        more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
        goto resume;
    }
 
    rb_econv_close(ec);
    return;
}
#else
/* sample transcode_loop implementation in byte-by-byte stream style */
static void
transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
               const unsigned char *in_stop, unsigned char *out_stop,
               VALUE destination,
               unsigned char *(*resize_destination)(VALUE, size_t, size_t),
               const char *src_encoding,
               const char *dst_encoding,
               int ecflags,
               VALUE ecopts)
{
    rb_econv_t *ec;
    rb_transcoding *last_tc;
    rb_econv_result_t ret;
    unsigned char *out_start = *out_pos;
    const unsigned char *ptr;
    int max_output;
    VALUE exc;
 
    ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
    if (!ec)
        rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
 
    last_tc = ec->last_tc;
    max_output = last_tc ? last_tc->transcoder->max_output : 1;
 
    ret = econv_source_buffer_empty;
    ptr = *in_pos;
    while (ret != econv_finished) {
        unsigned char input_byte;
        const unsigned char *p = &input_byte;
 
        if (ret == econv_source_buffer_empty) {
            if (ptr < in_stop) {
                input_byte = *ptr;
                ret = rb_econv_convert(ec, &p, p+1, out_pos, out_stop, ECONV_PARTIAL_INPUT);
            }
            else {
                ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, 0);
            }
        }
        else {
            ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, ECONV_PARTIAL_INPUT);
        }
        if (&input_byte != p)
            ptr += p - &input_byte;
        switch (ret) {
          case econv_invalid_byte_sequence:
          case econv_incomplete_input:
          case econv_undefined_conversion:
            exc = make_econv_exception(ec);
            rb_econv_close(ec);
            rb_exc_raise(exc);
            break;
 
          case econv_destination_buffer_full:
            more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
            break;
 
          case econv_source_buffer_empty:
            break;
 
          case econv_finished:
            break;
        }
    }
    rb_econv_close(ec);
    *in_pos = in_stop;
    return;
}
#endif
 
 
/*
 *  String-specific code
 */
 
static unsigned char *
str_transcoding_resize(VALUE destination, size_t len, size_t new_len)
{
    rb_str_resize(destination, new_len);
    return (unsigned char *)RSTRING_PTR(destination);
}
 
static int
econv_opts(VALUE opt, int ecflags)
{
    VALUE v;
    int newlineflag = 0;
 
    v = rb_hash_aref(opt, sym_invalid);
    if (NIL_P(v)) {
    }
    else if (v==sym_replace) {
        ecflags |= ECONV_INVALID_REPLACE;
    }
    else {
        rb_raise(rb_eArgError, "unknown value for invalid character option");
    }
 
    v = rb_hash_aref(opt, sym_undef);
    if (NIL_P(v)) {
    }
    else if (v==sym_replace) {
        ecflags |= ECONV_UNDEF_REPLACE;
    }
    else {
        rb_raise(rb_eArgError, "unknown value for undefined character option");
    }
 
    v = rb_hash_aref(opt, sym_replace);
    if (!NIL_P(v) && !(ecflags & ECONV_INVALID_REPLACE)) {
        ecflags |= ECONV_UNDEF_REPLACE;
    }
 
    v = rb_hash_aref(opt, sym_xml);
    if (!NIL_P(v)) {
        if (v==sym_text) {
            ecflags |= ECONV_XML_TEXT_DECORATOR|ECONV_UNDEF_HEX_CHARREF;
        }
        else if (v==sym_attr) {
            ecflags |= ECONV_XML_ATTR_CONTENT_DECORATOR|ECONV_XML_ATTR_QUOTE_DECORATOR|ECONV_UNDEF_HEX_CHARREF;
        }
        else if (SYMBOL_P(v)) {
            rb_raise(rb_eArgError, "unexpected value for xml option: %"PRIsVALUE, rb_sym2str(v));
        }
        else {
            rb_raise(rb_eArgError, "unexpected value for xml option");
        }
    }
 
#ifdef ENABLE_ECONV_NEWLINE_OPTION
    v = rb_hash_aref(opt, sym_newline);
    if (!NIL_P(v)) {
        newlineflag = 2;
        ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
        if (v == sym_universal) {
            ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR;
        }
        else if (v == sym_crlf) {
            ecflags |= ECONV_CRLF_NEWLINE_DECORATOR;
        }
        else if (v == sym_cr) {
            ecflags |= ECONV_CR_NEWLINE_DECORATOR;
        }
        else if (v == sym_lf) {
            ecflags |= ECONV_LF_NEWLINE_DECORATOR;
        }
        else if (SYMBOL_P(v)) {
            rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE,
                     rb_sym2str(v));
        }
        else {
            rb_raise(rb_eArgError, "unexpected value for newline option");
        }
    }
#endif
    {
        int setflags = 0;
 
        v = rb_hash_aref(opt, sym_universal_newline);
        if (RTEST(v))
            setflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR;
        newlineflag |= !NIL_P(v);
 
        v = rb_hash_aref(opt, sym_crlf_newline);
        if (RTEST(v))
            setflags |= ECONV_CRLF_NEWLINE_DECORATOR;
        newlineflag |= !NIL_P(v);
 
        v = rb_hash_aref(opt, sym_cr_newline);
        if (RTEST(v))
            setflags |= ECONV_CR_NEWLINE_DECORATOR;
        newlineflag |= !NIL_P(v);
 
        v = rb_hash_aref(opt, sym_lf_newline);
        if (RTEST(v))
            setflags |= ECONV_LF_NEWLINE_DECORATOR;
        newlineflag |= !NIL_P(v);
 
        switch (newlineflag) {
          case 1:
            ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
            ecflags |= setflags;
            break;
 
          case 3:
            rb_warning(":newline option precedes other newline options");
            break;
        }
    }
 
    return ecflags;
}
 
int
rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags)
{
    VALUE newhash = Qnil;
    VALUE v;
 
    if (NIL_P(opthash)) {
        *opts = Qnil;
        return ecflags;
    }
    ecflags = econv_opts(opthash, ecflags);
 
    v = rb_hash_aref(opthash, sym_replace);
    if (!NIL_P(v)) {
        StringValue(v);
        if (is_broken_string(v)) {
            VALUE dumped = rb_str_dump(v);
            rb_raise(rb_eArgError, "replacement string is broken: %s as %s",
                     StringValueCStr(dumped),
                     rb_enc_name(rb_enc_get(v)));
        }
        v = rb_str_new_frozen(v);
        newhash = rb_hash_new();
        rb_hash_aset(newhash, sym_replace, v);
    }
 
    v = rb_hash_aref(opthash, sym_fallback);
    if (!NIL_P(v)) {
        VALUE h = rb_check_hash_type(v);
        if (NIL_P(h)
            ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF))
            : (v = h, 1)) {
            if (NIL_P(newhash))
                newhash = rb_hash_new();
            rb_hash_aset(newhash, sym_fallback, v);
        }
    }
 
    if (!NIL_P(newhash))
        rb_hash_freeze(newhash);
    *opts = newhash;
 
    return ecflags;
}
 
int
rb_econv_prepare_opts(VALUE opthash, VALUE *opts)
{
    return rb_econv_prepare_options(opthash, opts, 0);
}
 
rb_econv_t *
rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE opthash)
{
    rb_econv_t *ec;
    VALUE replacement;
 
    if (NIL_P(opthash)) {
        replacement = Qnil;
    }
    else {
        if (!RB_TYPE_P(opthash, T_HASH) || !OBJ_FROZEN(opthash))
            rb_bug("rb_econv_open_opts called with invalid opthash");
        replacement = rb_hash_aref(opthash, sym_replace);
    }
 
    ec = rb_econv_open(source_encoding, destination_encoding, ecflags);
    if (ec) {
        if (!NIL_P(replacement)) {
            int ret;
            rb_encoding *enc = rb_enc_get(replacement);
 
            ret = rb_econv_set_replacement(ec,
                    (const unsigned char *)RSTRING_PTR(replacement),
                    RSTRING_LEN(replacement),
                    rb_enc_name(enc));
            if (ret == -1) {
                rb_econv_close(ec);
                ec = NULL;
            }
        }
    }
    return ec; // can be NULL
}
 
static int
enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p)
{
    rb_encoding *enc;
    const char *n;
    int encidx;
    VALUE encval;
 
    if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
        !(enc = rb_enc_from_index(encidx))) {
        enc = NULL;
        encidx = 0;
        n = StringValueCStr(*arg);
    }
    else {
        n = rb_enc_name(enc);
    }
 
    *name_p = n;
    *enc_p = enc;
 
    return encidx;
}
 
static int
str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2,
        const char **sname_p, rb_encoding **senc_p,
        const char **dname_p, rb_encoding **denc_p)
{
    rb_encoding *senc, *denc;
    const char *sname, *dname;
    int sencidx, dencidx;
 
    dencidx = enc_arg(arg1, &dname, &denc);
 
    if (NIL_P(*arg2)) {
        sencidx = rb_enc_get_index(str);
        senc = rb_enc_from_index(sencidx);
        sname = rb_enc_name(senc);
    }
    else {
        sencidx = enc_arg(arg2, &sname, &senc);
    }
 
    *sname_p = sname;
    *senc_p = senc;
    *dname_p = dname;
    *denc_p = denc;
    return dencidx;
}
 
static int
str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
{
    VALUE dest;
    VALUE str = *self;
    VALUE arg1, arg2;
    long blen, slen;
    unsigned char *buf, *bp, *sp;
    const unsigned char *fromp;
    rb_encoding *senc, *denc;
    const char *sname, *dname;
    int dencidx;
    int explicitly_invalid_replace = TRUE;
 
    rb_check_arity(argc, 0, 2);
 
    if (argc == 0) {
        arg1 = rb_enc_default_internal();
        if (NIL_P(arg1)) {
            if (!ecflags) return -1;
            arg1 = rb_obj_encoding(str);
        }
        if (!(ecflags & ECONV_INVALID_MASK)) {
            explicitly_invalid_replace = FALSE;
        }
        ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE;
    }
    else {
        arg1 = argv[0];
    }
    arg2 = argc<=1 ? Qnil : argv[1];
    dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
 
    if ((ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
                    ECONV_XML_TEXT_DECORATOR|
                    ECONV_XML_ATTR_CONTENT_DECORATOR|
                    ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) {
        if (senc && senc == denc) {
            if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) {
                VALUE rep = Qnil;
                if (!NIL_P(ecopts)) {
                    rep = rb_hash_aref(ecopts, sym_replace);
                }
                dest = rb_enc_str_scrub(senc, str, rep);
                if (NIL_P(dest)) dest = str;
                *self = dest;
                return dencidx;
            }
            return NIL_P(arg2) ? -1 : dencidx;
        }
        if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
            if (is_ascii_string(str)) {
                return dencidx;
            }
        }
        if (encoding_equal(sname, dname)) {
            return NIL_P(arg2) ? -1 : dencidx;
        }
    }
    else {
        if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) {
            rb_encoding *utf8 = rb_utf8_encoding();
            str = rb_str_conv_enc(str, senc, utf8);
            senc = utf8;
            sname = "UTF-8";
        }
        if (encoding_equal(sname, dname)) {
            sname = "";
            dname = "";
        }
    }
 
    fromp = sp = (unsigned char *)RSTRING_PTR(str);
    slen = RSTRING_LEN(str);
    blen = slen + 30; /* len + margin */
    dest = rb_str_tmp_new(blen);
    bp = (unsigned char *)RSTRING_PTR(dest);
 
    transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
    if (fromp != sp+slen) {
        rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
    }
    buf = (unsigned char *)RSTRING_PTR(dest);
    *bp = '\0';
    rb_str_set_len(dest, bp - buf);
 
    /* set encoding */
    if (!denc) {
        dencidx = rb_define_dummy_encoding(dname);
        RB_GC_GUARD(arg1);
        RB_GC_GUARD(arg2);
    }
    *self = dest;
 
    return dencidx;
}
 
static int
str_transcode(int argc, VALUE *argv, VALUE *self)
{
    VALUE opt;
    int ecflags = 0;
    VALUE ecopts = Qnil;
 
    argc = rb_scan_args(argc, argv, "02:", NULL, NULL, &opt);
    if (!NIL_P(opt)) {
        ecflags = rb_econv_prepare_opts(opt, &ecopts);
    }
    return str_transcode0(argc, argv, self, ecflags, ecopts);
}
 
static inline VALUE
str_encode_associate(VALUE str, int encidx)
{
    int cr = 0;
 
    rb_enc_associate_index(str, encidx);
 
    /* transcoded string never be broken. */
    if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
        rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr);
    }
    else {
        cr = ENC_CODERANGE_VALID;
    }
    ENC_CODERANGE_SET(str, cr);
    return str;
}
 
/*
 *  call-seq:
 *    encode!(dst_encoding = Encoding.default_internal, **enc_opts) -> self
 *    encode!(dst_encoding, src_encoding, **enc_opts)   -> self
 *
 *  Like #encode, but applies encoding changes to +self+; returns +self+.
 *
 *  Related: see {Modifying}[rdoc-ref:String@Modifying].
 */
 
static VALUE
str_encode_bang(int argc, VALUE *argv, VALUE str)
{
    VALUE newstr;
    int encidx;
 
    rb_check_frozen(str);
 
    newstr = str;
    encidx = str_transcode(argc, argv, &newstr);
 
    if (encidx < 0) return str;
    if (newstr == str) {
        rb_enc_associate_index(str, encidx);
        return str;
    }
    rb_str_shared_replace(str, newstr);
    return str_encode_associate(str, encidx);
}
 
static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx);
 
/*
 *  call-seq:
 *    encode(dst_encoding = Encoding.default_internal, **enc_opts) -> string
 *    encode(dst_encoding, src_encoding, **enc_opts)   -> string
 *
 *  :include: doc/string/encode.rdoc
 *
 */
 
static VALUE
str_encode(int argc, VALUE *argv, VALUE str)
{
    VALUE newstr = str;
    int encidx = str_transcode(argc, argv, &newstr);
    return encoded_dup(newstr, str, encidx);
}
 
VALUE
rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
{
    int argc = 1;
    VALUE *argv = &to;
    VALUE newstr = str;
    int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
    return encoded_dup(newstr, str, encidx);
}
 
static VALUE
encoded_dup(VALUE newstr, VALUE str, int encidx)
{
    if (encidx < 0) return rb_str_dup(str);
    if (newstr == str) {
        newstr = rb_str_dup(str);
        rb_enc_associate_index(newstr, encidx);
        return newstr;
    }
    else {
        RBASIC_SET_CLASS(newstr, rb_obj_class(str));
    }
    return str_encode_associate(newstr, encidx);
}
 
/*
 * Document-class: Encoding::Converter
 *
 * Encoding conversion class.
 */
static void
econv_free(void *ptr)
{
    rb_econv_t *ec = ptr;
    rb_econv_close(ec);
}
 
static size_t
econv_memsize(const void *ptr)
{
    return ptr ? rb_econv_memsize((rb_econv_t *)ptr) : 0;
}
 
static const rb_data_type_t econv_data_type = {
    "econv",
    {0, econv_free, econv_memsize,},
    0, 0, RUBY_TYPED_THREAD_SAFE_FREE
};
 
static VALUE
econv_s_allocate(VALUE klass)
{
    return TypedData_Wrap_Struct(klass, &econv_data_type, NULL);
}
 
static rb_encoding *
make_dummy_encoding(const char *name)
{
    rb_encoding *enc;
    int idx;
    idx = rb_define_dummy_encoding(name);
    enc = rb_enc_from_index(idx);
    return enc;
}
 
static rb_encoding *
make_encoding(const char *name)
{
    rb_encoding *enc;
    enc = rb_enc_find(name);
    if (!enc) {
        RB_VM_LOCKING() {
            if (rb_enc_registered(name)) {
                enc = NULL;
            }
            else {
                enc = make_dummy_encoding(name);
            }
        }
    }
    return enc;
}
 
static VALUE
make_encobj(const char *name)
{
    return rb_enc_from_encoding(make_encoding(name));
}
 
/*
 * call-seq:
 *   Encoding::Converter.asciicompat_encoding(string) -> encoding or nil
 *   Encoding::Converter.asciicompat_encoding(encoding) -> encoding or nil
 *
 * Returns the corresponding ASCII compatible encoding.
 *
 * Returns nil if the argument is an ASCII compatible encoding.
 *
 * "corresponding ASCII compatible encoding" is an ASCII compatible encoding which
 * can represents exactly the same characters as the given ASCII incompatible encoding.
 * So, no conversion undefined error occurs when converting between the two encodings.
 *
 *   Encoding::Converter.asciicompat_encoding("ISO-2022-JP") #=> #<Encoding:stateless-ISO-2022-JP>
 *   Encoding::Converter.asciicompat_encoding("UTF-16BE") #=> #<Encoding:UTF-8>
 *   Encoding::Converter.asciicompat_encoding("UTF-8") #=> nil
 *
 */
static VALUE
econv_s_asciicompat_encoding(VALUE klass, VALUE arg)
{
    const char *arg_name, *result_name;
    rb_encoding *arg_enc, *result_enc;
    VALUE enc = Qnil;
 
    enc_arg(&arg, &arg_name, &arg_enc);
    result_name = rb_econv_asciicompat_encoding(arg_name);
    if (result_name) {
        result_enc = make_encoding(result_name);
        enc = rb_enc_from_encoding(result_enc);
    }
    return enc;
}
 
static void
econv_args(int argc, VALUE *argv,
    VALUE *snamev_p, VALUE *dnamev_p,
    const char **sname_p, const char **dname_p,
    rb_encoding **senc_p, rb_encoding **denc_p,
    int *ecflags_p,
    VALUE *ecopts_p)
{
    VALUE opt, flags_v, ecopts;
    int sidx, didx;
    const char *sname, *dname;
    rb_encoding *senc, *denc;
    int ecflags;
 
    argc = rb_scan_args(argc, argv, "21:", snamev_p, dnamev_p, &flags_v, &opt);
 
    if (!NIL_P(flags_v)) {
        if (!NIL_P(opt)) {
            rb_error_arity(argc + 1, 2, 3);
        }
        ecflags = NUM2INT(rb_to_int(flags_v));
        ecopts = Qnil;
    }
    else if (!NIL_P(opt)) {
        ecflags = rb_econv_prepare_opts(opt, &ecopts);
    }
    else {
        ecflags = 0;
        ecopts = Qnil;
    }
 
    senc = NULL;
    sidx = rb_to_encoding_index(*snamev_p);
    if (0 <= sidx) {
        senc = rb_enc_from_index(sidx);
    }
    else {
        StringValue(*snamev_p);
    }
 
    denc = NULL;
    didx = rb_to_encoding_index(*dnamev_p);
    if (0 <= didx) {
        denc = rb_enc_from_index(didx);
    }
    else {
        StringValue(*dnamev_p);
    }
 
    sname = senc ? rb_enc_name(senc) : StringValueCStr(*snamev_p);
    dname = denc ? rb_enc_name(denc) : StringValueCStr(*dnamev_p);
 
    *sname_p = sname;
    *dname_p = dname;
    *senc_p = senc;
    *denc_p = denc;
    *ecflags_p = ecflags;
    *ecopts_p = ecopts;
}
 
static int
decorate_convpath(VALUE convpath, int ecflags)
{
    int num_decorators;
    const char *decorators[MAX_ECFLAGS_DECORATORS];
    int i;
    int n, len;
 
    num_decorators = decorator_names(ecflags, decorators);
    if (num_decorators == -1)
        return -1;
 
    len = n = RARRAY_LENINT(convpath);
    if (n != 0) {
        VALUE pair = RARRAY_AREF(convpath, n-1);
        if (RB_TYPE_P(pair, T_ARRAY)) {
            const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0)));
            const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1)));
            transcoder_entry_t *entry;
            const rb_transcoder *tr;
            entry = get_transcoder_entry(sname, dname);
            tr = load_transcoder_entry(entry);
            if (!tr)
                return -1;
            if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
                    tr->asciicompat_type == asciicompat_encoder) {
                n--;
                rb_ary_store(convpath, len + num_decorators - 1, pair);
            }
        }
        else {
            rb_ary_store(convpath, len + num_decorators - 1, pair);
        }
    }
 
    for (i = 0; i < num_decorators; i++)
        rb_ary_store(convpath, n + i, rb_str_new_cstr(decorators[i]));
 
    return 0;
}
 
static void
search_convpath_i(const char *sname, const char *dname, int depth, void *arg)
{
    VALUE *ary_p = arg;
    VALUE v;
 
    if (NIL_P(*ary_p)) {
        *ary_p = rb_ary_new();
    }
 
    if (DECORATOR_P(sname, dname)) {
        v = rb_str_new_cstr(dname);
    }
    else {
        v = rb_assoc_new(make_encobj(sname), make_encobj(dname));
    }
    rb_ary_store(*ary_p, depth, v);
}
 
/*
 * call-seq:
 *   Encoding::Converter.search_convpath(source_encoding, destination_encoding)         -> ary
 *   Encoding::Converter.search_convpath(source_encoding, destination_encoding, opt)    -> ary
 *
 *  Returns a conversion path.
 *
 *   p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP")
 *   #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
 *   #    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>]]
 *
 *   p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", universal_newline: true)
 *   or
 *   p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", newline: :universal)
 *   #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
 *   #    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
 *   #    "universal_newline"]
 *
 *   p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", universal_newline: true)
 *   or
 *   p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", newline: :universal)
 *   #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
 *   #    "universal_newline",
 *   #    [#<Encoding:UTF-8>, #<Encoding:UTF-32BE>]]
 */
static VALUE
econv_s_search_convpath(int argc, VALUE *argv, VALUE klass)
{
    VALUE snamev, dnamev;
    const char *sname, *dname;
    rb_encoding *senc, *denc;
    int ecflags;
    VALUE ecopts;
    VALUE convpath;
 
    econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
 
    convpath = Qnil;
    transcode_search_path(sname, dname, search_convpath_i, &convpath);
 
    if (NIL_P(convpath)) {
        VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
        RB_GC_GUARD(snamev);
        RB_GC_GUARD(dnamev);
        rb_exc_raise(exc);
    }
 
    if (decorate_convpath(convpath, ecflags) == -1) {
        VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
        RB_GC_GUARD(snamev);
        RB_GC_GUARD(dnamev);
        rb_exc_raise(exc);
    }
 
    return convpath;
}
 
/*
 * Check the existence of a conversion path.
 * Returns the number of converters in the conversion path.
 * result: >=0:success -1:failure
 */
int
rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding)
{
    VALUE convpath = Qnil;
    transcode_search_path(from_encoding, to_encoding, search_convpath_i,
                          &convpath);
    return RTEST(convpath);
}
 
struct rb_econv_init_by_convpath_t {
    rb_econv_t *ec;
    int index;
    int ret;
};
 
static void
rb_econv_init_by_convpath_i(const char *sname, const char *dname, int depth, void *arg)
{
    struct rb_econv_init_by_convpath_t *a = (struct rb_econv_init_by_convpath_t *)arg;
    int ret;
 
    if (a->ret == -1)
        return;
 
    ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
 
    a->ret = ret;
    return;
}
 
static rb_econv_t *
rb_econv_init_by_convpath(VALUE self, VALUE convpath,
    const char **sname_p, const char **dname_p,
    rb_encoding **senc_p, rb_encoding**denc_p)
{
    rb_econv_t *ec;
    long i;
    int ret, first=1;
    VALUE elt;
    rb_encoding *senc = 0, *denc = 0;
    const char *sname, *dname;
 
    ec = rb_econv_alloc(RARRAY_LENINT(convpath));
    DATA_PTR(self) = ec;
 
    for (i = 0; i < RARRAY_LEN(convpath); i++) {
        VALUE snamev, dnamev;
        VALUE pair;
        elt = rb_ary_entry(convpath, i);
        if (!NIL_P(pair = rb_check_array_type(elt))) {
            if (RARRAY_LEN(pair) != 2)
                rb_raise(rb_eArgError, "not a 2-element array in convpath");
            snamev = rb_ary_entry(pair, 0);
            enc_arg(&snamev, &sname, &senc);
            dnamev = rb_ary_entry(pair, 1);
            enc_arg(&dnamev, &dname, &denc);
        }
        else {
            sname = "";
            dname = StringValueCStr(elt);
        }
        if (DECORATOR_P(sname, dname)) {
            ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
            if (ret == -1) {
                VALUE msg = rb_sprintf("decoration failed: %s", dname);
                RB_GC_GUARD(snamev);
                RB_GC_GUARD(dnamev);
                rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
            }
        }
        else {
            int j = ec->num_trans;
            struct rb_econv_init_by_convpath_t arg;
            arg.ec = ec;
            arg.index = ec->num_trans;
            arg.ret = 0;
            ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
            if (ret == -1 || arg.ret == -1) {
                VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname);
                RB_GC_GUARD(snamev);
                RB_GC_GUARD(dnamev);
                rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
            }
            if (first) {
                first = 0;
                *senc_p = senc;
                *sname_p = ec->elems[j].tc->transcoder->src_encoding;
            }
            *denc_p = denc;
            *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
        }
    }
 
    if (first) {
        *senc_p = NULL;
        *denc_p = NULL;
        *sname_p = "";
        *dname_p = "";
    }
 
    ec->source_encoding_name = *sname_p;
    ec->destination_encoding_name = *dname_p;
 
    return ec;
}
 
/*
 * call-seq:
 *   Encoding::Converter.new(source_encoding, destination_encoding)
 *   Encoding::Converter.new(source_encoding, destination_encoding, opt)
 *   Encoding::Converter.new(convpath)
 *
 * possible options elements:
 *   hash form:
 *     :invalid => nil            # raise error on invalid byte sequence (default)
 *     :invalid => :replace       # replace invalid byte sequence
 *     :undef => nil              # raise error on undefined conversion (default)
 *     :undef => :replace         # replace undefined conversion
 *     :replace => string         # replacement string ("?" or "\uFFFD" if not specified)
 *     :newline => :universal     # decorator for converting CRLF and CR to LF
 *     :newline => :lf            # decorator for converting CRLF and CR to LF when writing
 *     :newline => :crlf          # decorator for converting LF to CRLF
 *     :newline => :cr            # decorator for converting LF to CR
 *     :universal_newline => true # decorator for converting CRLF and CR to LF
 *     :crlf_newline => true      # decorator for converting LF to CRLF
 *     :cr_newline => true        # decorator for converting LF to CR
 *     :lf_newline => true        # decorator for converting CRLF and CR to LF when writing
 *     :xml => :text              # escape as XML CharData.
 *     :xml => :attr              # escape as XML AttValue
 *   integer form:
 *     Encoding::Converter::INVALID_REPLACE
 *     Encoding::Converter::UNDEF_REPLACE
 *     Encoding::Converter::UNDEF_HEX_CHARREF
 *     Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR
 *     Encoding::Converter::LF_NEWLINE_DECORATOR
 *     Encoding::Converter::CRLF_NEWLINE_DECORATOR
 *     Encoding::Converter::CR_NEWLINE_DECORATOR
 *     Encoding::Converter::XML_TEXT_DECORATOR
 *     Encoding::Converter::XML_ATTR_CONTENT_DECORATOR
 *     Encoding::Converter::XML_ATTR_QUOTE_DECORATOR
 *
 * Encoding::Converter.new creates an instance of Encoding::Converter.
 *
 * Source_encoding and destination_encoding should be a string or
 * Encoding object.
 *
 * opt should be nil, a hash or an integer.
 *
 * convpath should be an array.
 * convpath may contain
 * - two-element arrays which contain encodings or encoding names, or
 * - strings representing decorator names.
 *
 * Encoding::Converter.new optionally takes an option.
 * The option should be a hash or an integer.
 * The option hash can contain :invalid => nil, etc.
 * The option integer should be logical-or of constants such as
 * Encoding::Converter::INVALID_REPLACE, etc.
 *
 * [:invalid => nil]
 *   Raise error on invalid byte sequence.  This is a default behavior.
 * [:invalid => :replace]
 *   Replace invalid byte sequence by replacement string.
 * [:undef => nil]
 *   Raise an error if a character in source_encoding is not defined in destination_encoding.
 *   This is a default behavior.
 * [:undef => :replace]
 *   Replace undefined character in destination_encoding with replacement string.
 * [:replace => string]
 *   Specify the replacement string.
 *   If not specified, "\uFFFD" is used for Unicode encodings and "?" for others.
 * [:universal_newline => true]
 *   Convert CRLF and CR to LF.
 * [:crlf_newline => true]
 *   Convert LF to CRLF.
 * [:cr_newline => true]
 *   Convert LF to CR.
 * [:lf_newline => true]
 *   Convert CRLF and CR to LF (when writing).
 * [:xml => :text]
 *   Escape as XML CharData.
 *   This form can be used as an HTML 4.0 #PCDATA.
 *   - '&' -> '&amp;'
 *   - '<' -> '&lt;'
 *   - '>' -> '&gt;'
 *   - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
 * [:xml => :attr]
 *   Escape as XML AttValue.
 *   The converted result is quoted as "...".
 *   This form can be used as an HTML 4.0 attribute value.
 *   - '&' -> '&amp;'
 *   - '<' -> '&lt;'
 *   - '>' -> '&gt;'
 *   - '"' -> '&quot;'
 *   - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
 *
 * Examples:
 *   # UTF-16BE to UTF-8
 *   ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
 *
 *   # Usually, decorators such as newline conversion are inserted last.
 *   ec = Encoding::Converter.new("UTF-16BE", "UTF-8", :universal_newline => true)
 *   p ec.convpath #=> [[#<Encoding:UTF-16BE>, #<Encoding:UTF-8>],
 *                 #    "universal_newline"]
 *
 *   # But, if the last encoding is ASCII incompatible,
 *   # decorators are inserted before the last conversion.
 *   ec = Encoding::Converter.new("UTF-8", "UTF-16BE", :crlf_newline => true)
 *   p ec.convpath #=> ["crlf_newline",
 *                 #    [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
 *
 *   # Conversion path can be specified directly.
 *   ec = Encoding::Converter.new(["universal_newline", ["EUC-JP", "UTF-8"], ["UTF-8", "UTF-16BE"]])
 *   p ec.convpath #=> ["universal_newline",
 *                 #    [#<Encoding:EUC-JP>, #<Encoding:UTF-8>],
 *                 #    [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
 */
static VALUE
econv_init(int argc, VALUE *argv, VALUE self)
{
    VALUE ecopts;
    VALUE snamev, dnamev;
    const char *sname, *dname;
    rb_encoding *senc, *denc;
    rb_econv_t *ec;
    int ecflags;
    VALUE convpath;
 
    if (rb_check_typeddata(self, &econv_data_type)) {
        rb_raise(rb_eTypeError, "already initialized");
    }
 
    if (argc == 1 && !NIL_P(convpath = rb_check_array_type(argv[0]))) {
        ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
        ecflags = 0;
        ecopts = Qnil;
    }
    else {
        econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
        ec = rb_econv_open_opts(sname, dname, ecflags, ecopts);
    }
 
    if (!ec) {
        VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
        RB_GC_GUARD(snamev);
        RB_GC_GUARD(dnamev);
        rb_exc_raise(exc);
    }
 
    if (!DECORATOR_P(sname, dname)) {
        if (!senc)
            senc = make_dummy_encoding(sname);
        if (!denc)
            denc = make_dummy_encoding(dname);
        RB_GC_GUARD(snamev);
        RB_GC_GUARD(dnamev);
    }
 
    ec->source_encoding = senc;
    ec->destination_encoding = denc;
 
    DATA_PTR(self) = ec;
 
    return self;
}
 
/*
 * call-seq:
 *   ec.inspect         -> string
 *
 * Returns a printable version of <i>ec</i>
 *
 *   ec = Encoding::Converter.new("iso-8859-1", "utf-8")
 *   puts ec.inspect    #=> #<Encoding::Converter: ISO-8859-1 to UTF-8>
 *
 */
static VALUE
econv_inspect(VALUE self)
{
    const char *cname = rb_obj_classname(self);
    rb_econv_t *ec;
 
    TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
    if (!ec)
        return rb_sprintf("#<%s: uninitialized>", cname);
    else {
        const char *sname = ec->source_encoding_name;
        const char *dname = ec->destination_encoding_name;
        VALUE str;
        str = rb_sprintf("#<%s: ", cname);
        econv_description(sname, dname, ec->flags, str);
        rb_str_cat2(str, ">");
        return str;
    }
}
 
static rb_econv_t *
check_econv(VALUE self)
{
    rb_econv_t *ec;
 
    TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
    if (!ec) {
        rb_raise(rb_eTypeError, "uninitialized encoding converter");
    }
    return ec;
}
 
static VALUE
econv_get_encoding(rb_encoding *encoding)
{
    if (!encoding)
        return Qnil;
    return rb_enc_from_encoding(encoding);
}
 
/*
 * call-seq:
 *   ec.source_encoding -> encoding
 *
 * Returns the source encoding as an Encoding object.
 */
static VALUE
econv_source_encoding(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    return econv_get_encoding(ec->source_encoding);
}
 
/*
 * call-seq:
 *   ec.destination_encoding -> encoding
 *
 * Returns the destination encoding as an Encoding object.
 */
static VALUE
econv_destination_encoding(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    return econv_get_encoding(ec->destination_encoding);
}
 
/*
 * call-seq:
 *   ec.convpath        -> ary
 *
 * Returns the conversion path of ec.
 *
 * The result is an array of conversions.
 *
 *   ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP", crlf_newline: true)
 *   p ec.convpath
 *   #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
 *   #    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
 *   #    "crlf_newline"]
 *
 * Each element of the array is a pair of encodings or a string.
 * A pair means an encoding conversion.
 * A string means a decorator.
 *
 * In the above example, [#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>] means
 * a converter from ISO-8859-1 to UTF-8.
 * "crlf_newline" means newline converter from LF to CRLF.
 */
static VALUE
econv_convpath(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    VALUE result;
    int i;
 
    result = rb_ary_new();
    for (i = 0; i < ec->num_trans; i++) {
        const rb_transcoder *tr = ec->elems[i].tc->transcoder;
        VALUE v;
        if (DECORATOR_P(tr->src_encoding, tr->dst_encoding))
            v = rb_str_new_cstr(tr->dst_encoding);
        else
            v = rb_assoc_new(make_encobj(tr->src_encoding), make_encobj(tr->dst_encoding));
        rb_ary_push(result, v);
    }
    return result;
}
 
/*
 * call-seq:
 *   ec == other        -> true or false
 */
static VALUE
econv_equal(VALUE self, VALUE other)
{
    rb_econv_t *ec1 = check_econv(self);
    rb_econv_t *ec2;
    int i;
 
    if (!rb_typeddata_is_kind_of(other, &econv_data_type)) {
        return Qnil;
    }
    ec2 = DATA_PTR(other);
    if (!ec2) return Qfalse;
    if (ec1->source_encoding_name != ec2->source_encoding_name &&
        strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
        return Qfalse;
    if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
        strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
        return Qfalse;
    if (ec1->flags != ec2->flags) return Qfalse;
    if (ec1->replacement_enc != ec2->replacement_enc &&
        strcmp(ec1->replacement_enc, ec2->replacement_enc))
        return Qfalse;
    if (ec1->replacement_len != ec2->replacement_len) return Qfalse;
    if (ec1->replacement_str != ec2->replacement_str &&
        memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
        return Qfalse;
 
    if (ec1->num_trans != ec2->num_trans) return Qfalse;
    for (i = 0; i < ec1->num_trans; i++) {
        if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
            return Qfalse;
    }
    return Qtrue;
}
 
static VALUE
econv_result_to_symbol(rb_econv_result_t res)
{
    switch (res) {
      case econv_invalid_byte_sequence: return sym_invalid_byte_sequence;
      case econv_incomplete_input: return sym_incomplete_input;
      case econv_undefined_conversion: return sym_undefined_conversion;
      case econv_destination_buffer_full: return sym_destination_buffer_full;
      case econv_source_buffer_empty: return sym_source_buffer_empty;
      case econv_finished: return sym_finished;
      case econv_after_output: return sym_after_output;
      default: return INT2NUM(res); /* should not be reached */
    }
}
 
/*
 * call-seq:
 *   ec.primitive_convert(source_buffer, destination_buffer) -> symbol
 *   ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset) -> symbol
 *   ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) -> symbol
 *   ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, opt) -> symbol
 *
 * possible opt elements:
 *   hash form:
 *     :partial_input => true           # source buffer may be part of larger source
 *     :after_output => true            # stop conversion after output before input
 *   integer form:
 *     Encoding::Converter::PARTIAL_INPUT
 *     Encoding::Converter::AFTER_OUTPUT
 *
 * possible results:
 *    :invalid_byte_sequence
 *    :incomplete_input
 *    :undefined_conversion
 *    :after_output
 *    :destination_buffer_full
 *    :source_buffer_empty
 *    :finished
 *
 * primitive_convert converts source_buffer into destination_buffer.
 *
 * source_buffer should be a string or nil.
 * nil means an empty string.
 *
 * destination_buffer should be a string.
 *
 * destination_byteoffset should be an integer or nil.
 * nil means the end of destination_buffer.
 * If it is omitted, nil is assumed.
 *
 * destination_bytesize should be an integer or nil.
 * nil means unlimited.
 * If it is omitted, nil is assumed.
 *
 * opt should be nil, a hash or an integer.
 * nil means no flags.
 * If it is omitted, nil is assumed.
 *
 * primitive_convert converts the content of source_buffer from beginning
 * and store the result into destination_buffer.
 *
 * destination_byteoffset and destination_bytesize specify the region which
 * the converted result is stored.
 * destination_byteoffset specifies the start position in destination_buffer in bytes.
 * If destination_byteoffset is nil,
 * destination_buffer.bytesize is used for appending the result.
 * destination_bytesize specifies maximum number of bytes.
 * If destination_bytesize is nil,
 * destination size is unlimited.
 * After conversion, destination_buffer is resized to
 * destination_byteoffset + actually produced number of bytes.
 * Also destination_buffer's encoding is set to destination_encoding.
 *
 * primitive_convert drops the converted part of source_buffer.
 * the dropped part is converted in destination_buffer or
 * buffered in Encoding::Converter object.
 *
 * primitive_convert stops conversion when one of following condition met.
 * - invalid byte sequence found in source buffer (:invalid_byte_sequence)
 *   +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
 * - unexpected end of source buffer (:incomplete_input)
 *   this occur only when :partial_input is not specified.
 *   +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
 * - character not representable in output encoding (:undefined_conversion)
 *   +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
 * - after some output is generated, before input is done (:after_output)
 *   this occur only when :after_output is specified.
 * - destination buffer is full (:destination_buffer_full)
 *   this occur only when destination_bytesize is non-nil.
 * - source buffer is empty (:source_buffer_empty)
 *   this occur only when :partial_input is specified.
 * - conversion is finished (:finished)
 *
 * example:
 *   ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
 *   ret = ec.primitive_convert(src="pi", dst="", nil, 100)
 *   p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"]
 *
 *   ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
 *   ret = ec.primitive_convert(src="pi", dst="", nil, 1)
 *   p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"]
 *   ret = ec.primitive_convert(src, dst="", nil, 1)
 *   p [ret, src, dst] #=> [:destination_buffer_full, "", "p"]
 *   ret = ec.primitive_convert(src, dst="", nil, 1)
 *   p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"]
 *   ret = ec.primitive_convert(src, dst="", nil, 1)
 *   p [ret, src, dst] #=> [:finished, "", "i"]
 *
 */
static VALUE
econv_primitive_convert(int argc, VALUE *argv, VALUE self)
{
    VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
    rb_econv_t *ec = check_econv(self);
    rb_econv_result_t res;
    const unsigned char *ip, *is;
    unsigned char *op, *os;
    long output_byteoffset, output_bytesize;
    unsigned long output_byteend;
    int flags;
 
    argc = rb_scan_args(argc, argv, "23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
 
    if (NIL_P(output_byteoffset_v))
        output_byteoffset = 0; /* dummy */
    else
        output_byteoffset = NUM2LONG(output_byteoffset_v);
 
    if (NIL_P(output_bytesize_v))
        output_bytesize = 0; /* dummy */
    else
        output_bytesize = NUM2LONG(output_bytesize_v);
 
    if (!NIL_P(flags_v)) {
        if (!NIL_P(opt)) {
            rb_error_arity(argc + 1, 2, 5);
        }
        flags = NUM2INT(rb_to_int(flags_v));
    }
    else if (!NIL_P(opt)) {
        VALUE v;
        flags = 0;
        v = rb_hash_aref(opt, sym_partial_input);
        if (RTEST(v))
            flags |= ECONV_PARTIAL_INPUT;
        v = rb_hash_aref(opt, sym_after_output);
        if (RTEST(v))
            flags |= ECONV_AFTER_OUTPUT;
    }
    else {
        flags = 0;
    }
 
    StringValue(output);
    if (!NIL_P(input))
        StringValue(input);
    rb_str_modify(output);
 
    if (NIL_P(output_bytesize_v)) {
        output_bytesize = rb_str_capacity(output);
 
        if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input))
            output_bytesize = RSTRING_LEN(input);
    }
 
  retry:
 
    if (NIL_P(output_byteoffset_v))
        output_byteoffset = RSTRING_LEN(output);
 
    if (output_byteoffset < 0)
        rb_raise(rb_eArgError, "negative output_byteoffset");
 
    if (RSTRING_LEN(output) < output_byteoffset)
        rb_raise(rb_eArgError, "output_byteoffset too big");
 
    if (output_bytesize < 0)
        rb_raise(rb_eArgError, "negative output_bytesize");
 
    output_byteend = (unsigned long)output_byteoffset +
                     (unsigned long)output_bytesize;
 
    if (output_byteend < (unsigned long)output_byteoffset ||
        LONG_MAX < output_byteend)
        rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big");
 
    if (rb_str_capacity(output) < output_byteend)
        rb_str_resize(output, output_byteend);
 
    if (NIL_P(input)) {
        ip = is = NULL;
    }
    else {
        ip = (const unsigned char *)RSTRING_PTR(input);
        is = ip + RSTRING_LEN(input);
    }
 
    op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset;
    os = op + output_bytesize;
 
    res = rb_econv_convert(ec, &ip, is, &op, os, flags);
    rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output));
    if (!NIL_P(input)) {
        rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input));
    }
 
    if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) {
        if (LONG_MAX / 2 < output_bytesize)
            rb_raise(rb_eArgError, "too long conversion result");
        output_bytesize *= 2;
        output_byteoffset_v = Qnil;
        goto retry;
    }
 
    if (ec->destination_encoding) {
        rb_enc_associate(output, ec->destination_encoding);
    }
 
    return econv_result_to_symbol(res);
}
 
/*
 * call-seq:
 *   ec.convert(source_string) -> destination_string
 *
 * Convert source_string and return destination_string.
 *
 * source_string is assumed as a part of source.
 * i.e.  :partial_input=>true is specified internally.
 * finish method should be used last.
 *
 *   ec = Encoding::Converter.new("utf-8", "euc-jp")
 *   puts ec.convert("\u3042").dump     #=> "\xA4\xA2"
 *   puts ec.finish.dump                #=> ""
 *
 *   ec = Encoding::Converter.new("euc-jp", "utf-8")
 *   puts ec.convert("\xA4").dump       #=> ""
 *   puts ec.convert("\xA2").dump       #=> "\xE3\x81\x82"
 *   puts ec.finish.dump                #=> ""
 *
 *   ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
 *   puts ec.convert("\xE3").dump       #=> "".force_encoding("ISO-2022-JP")
 *   puts ec.convert("\x81").dump       #=> "".force_encoding("ISO-2022-JP")
 *   puts ec.convert("\x82").dump       #=> "\e$B$\"".force_encoding("ISO-2022-JP")
 *   puts ec.finish.dump                #=> "\e(B".force_encoding("ISO-2022-JP")
 *
 * If a conversion error occur,
 * Encoding::UndefinedConversionError or
 * Encoding::InvalidByteSequenceError is raised.
 * Encoding::Converter#convert doesn't supply methods to recover or restart
 * from these exceptions.
 * When you want to handle these conversion errors,
 * use Encoding::Converter#primitive_convert.
 *
 */
static VALUE
econv_convert(VALUE self, VALUE source_string)
{
    VALUE ret, dst;
    VALUE av[5];
    int ac;
    rb_econv_t *ec = check_econv(self);
 
    StringValue(source_string);
 
    dst = rb_str_new(NULL, 0);
 
    av[0] = rb_str_dup(source_string);
    av[1] = dst;
    av[2] = Qnil;
    av[3] = Qnil;
    av[4] = INT2NUM(ECONV_PARTIAL_INPUT);
    ac = 5;
 
    ret = econv_primitive_convert(ac, av, self);
 
    if (ret == sym_invalid_byte_sequence ||
        ret == sym_undefined_conversion ||
        ret == sym_incomplete_input) {
        VALUE exc = make_econv_exception(ec);
        rb_exc_raise(exc);
    }
 
    if (ret == sym_finished) {
        rb_raise(rb_eArgError, "converter already finished");
    }
 
    if (ret != sym_source_buffer_empty) {
        rb_bug("unexpected result of econv_primitive_convert");
    }
 
    return dst;
}
 
/*
 * call-seq:
 *   ec.finish -> string
 *
 * Finishes the converter.
 * It returns the last part of the converted string.
 *
 *   ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
 *   p ec.convert("\u3042")     #=> "\e$B$\""
 *   p ec.finish                #=> "\e(B"
 */
static VALUE
econv_finish(VALUE self)
{
    VALUE ret, dst;
    VALUE av[5];
    int ac;
    rb_econv_t *ec = check_econv(self);
 
    dst = rb_str_new(NULL, 0);
 
    av[0] = Qnil;
    av[1] = dst;
    av[2] = Qnil;
    av[3] = Qnil;
    av[4] = INT2FIX(0);
    ac = 5;
 
    ret = econv_primitive_convert(ac, av, self);
 
    if (ret == sym_invalid_byte_sequence ||
        ret == sym_undefined_conversion ||
        ret == sym_incomplete_input) {
        VALUE exc = make_econv_exception(ec);
        rb_exc_raise(exc);
    }
 
    if (ret != sym_finished) {
        rb_bug("unexpected result of econv_primitive_convert");
    }
 
    return dst;
}
 
/*
 * call-seq:
 *   ec.primitive_errinfo -> array
 *
 * primitive_errinfo returns important information regarding the last error
 * as a 5-element array:
 *
 *   [result, enc1, enc2, error_bytes, readagain_bytes]
 *
 * result is the last result of primitive_convert.
 *
 * Other elements are only meaningful when result is
 * :invalid_byte_sequence, :incomplete_input or :undefined_conversion.
 *
 * enc1 and enc2 indicate a conversion step as a pair of strings.
 * For example, a converter from EUC-JP to ISO-8859-1 converts
 * a string as follows: EUC-JP -> UTF-8 -> ISO-8859-1.
 * So [enc1, enc2] is either ["EUC-JP", "UTF-8"] or ["UTF-8", "ISO-8859-1"].
 *
 * error_bytes and readagain_bytes indicate the byte sequences which caused the error.
 * error_bytes is discarded portion.
 * readagain_bytes is buffered portion which is read again on next conversion.
 *
 * Example:
 *
 *   # \xff is invalid as EUC-JP.
 *   ec = Encoding::Converter.new("EUC-JP", "Shift_JIS")
 *   ec.primitive_convert(src="\xff", dst="", nil, 10)
 *   p ec.primitive_errinfo
 *   #=> [:invalid_byte_sequence, "EUC-JP", "Shift_JIS", "\xFF", ""]
 *
 *   # HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1.
 *   # Since this error is occur in UTF-8 to ISO-8859-1 conversion,
 *   # error_bytes is HIRAGANA LETTER A in UTF-8 (\xE3\x81\x82).
 *   ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
 *   ec.primitive_convert(src="\xa4\xa2", dst="", nil, 10)
 *   p ec.primitive_errinfo
 *   #=> [:undefined_conversion, "UTF-8", "ISO-8859-1", "\xE3\x81\x82", ""]
 *
 *   # partial character is invalid
 *   ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
 *   ec.primitive_convert(src="\xa4", dst="", nil, 10)
 *   p ec.primitive_errinfo
 *   #=> [:incomplete_input, "EUC-JP", "UTF-8", "\xA4", ""]
 *
 *   # Encoding::Converter::PARTIAL_INPUT prevents invalid errors by
 *   # partial characters.
 *   ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
 *   ec.primitive_convert(src="\xa4", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT)
 *   p ec.primitive_errinfo
 *   #=> [:source_buffer_empty, nil, nil, nil, nil]
 *
 *   # \xd8\x00\x00@ is invalid as UTF-16BE because
 *   # no low surrogate after high surrogate (\xd8\x00).
 *   # It is detected by 3rd byte (\00) which is part of next character.
 *   # So the high surrogate (\xd8\x00) is discarded and
 *   # the 3rd byte is read again later.
 *   # Since the byte is buffered in ec, it is dropped from src.
 *   ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
 *   ec.primitive_convert(src="\xd8\x00\x00@", dst="", nil, 10)
 *   p ec.primitive_errinfo
 *   #=> [:invalid_byte_sequence, "UTF-16BE", "UTF-8", "\xD8\x00", "\x00"]
 *   p src
 *   #=> "@"
 *
 *   # Similar to UTF-16BE, \x00\xd8@\x00 is invalid as UTF-16LE.
 *   # The problem is detected by 4th byte.
 *   ec = Encoding::Converter.new("UTF-16LE", "UTF-8")
 *   ec.primitive_convert(src="\x00\xd8@\x00", dst="", nil, 10)
 *   p ec.primitive_errinfo
 *   #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "@\x00"]
 *   p src
 *   #=> ""
 *
 */
static VALUE
econv_primitive_errinfo(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
 
    VALUE ary;
 
    ary = rb_ary_new2(5);
 
    rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
    rb_ary_store(ary, 4, Qnil);
 
    if (ec->last_error.source_encoding)
        rb_ary_store(ary, 1, rb_str_new2(ec->last_error.source_encoding));
 
    if (ec->last_error.destination_encoding)
        rb_ary_store(ary, 2, rb_str_new2(ec->last_error.destination_encoding));
 
    if (ec->last_error.error_bytes_start) {
        rb_ary_store(ary, 3, rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
        rb_ary_store(ary, 4, rb_str_new((const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
    }
 
    return ary;
}
 
/*
 * call-seq:
 *   ec.insert_output(string) -> nil
 *
 * Inserts string into the encoding converter.
 * The string will be converted to the destination encoding and
 * output on later conversions.
 *
 * If the destination encoding is stateful,
 * string is converted according to the state and the state is updated.
 *
 * This method should be used only when a conversion error occurs.
 *
 *  ec = Encoding::Converter.new("utf-8", "iso-8859-1")
 *  src = "HIRAGANA LETTER A is \u{3042}."
 *  dst = ""
 *  p ec.primitive_convert(src, dst)    #=> :undefined_conversion
 *  puts "[#{dst.dump}, #{src.dump}]"   #=> ["HIRAGANA LETTER A is ", "."]
 *  ec.insert_output("<err>")
 *  p ec.primitive_convert(src, dst)    #=> :finished
 *  puts "[#{dst.dump}, #{src.dump}]"   #=> ["HIRAGANA LETTER A is <err>.", ""]
 *
 *  ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
 *  src = "\u{306F 3041 3068 2661 3002}" # U+2661 is not representable in iso-2022-jp
 *  dst = ""
 *  p ec.primitive_convert(src, dst)    #=> :undefined_conversion
 *  puts "[#{dst.dump}, #{src.dump}]"   #=> ["\e$B$O$!$H".force_encoding("ISO-2022-JP"), "\xE3\x80\x82"]
 *  ec.insert_output "?"                # state change required to output "?".
 *  p ec.primitive_convert(src, dst)    #=> :finished
 *  puts "[#{dst.dump}, #{src.dump}]"   #=> ["\e$B$O$!$H\e(B?\e$B!#\e(B".force_encoding("ISO-2022-JP"), ""]
 *
 */
static VALUE
econv_insert_output(VALUE self, VALUE string)
{
    const char *insert_enc;
 
    int ret;
 
    rb_econv_t *ec = check_econv(self);
 
    StringValue(string);
    insert_enc = rb_econv_encoding_to_insert_output(ec);
    string = rb_str_encode(string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0, Qnil);
 
    ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc);
    if (ret == -1) {
        rb_raise(rb_eArgError, "too big string");
    }
 
    return Qnil;
}
 
/*
 * call-seq:
 *   ec.putback                    -> string
 *   ec.putback(max_numbytes)      -> string
 *
 * Put back the bytes which will be converted.
 *
 * The bytes are caused by invalid_byte_sequence error.
 * When invalid_byte_sequence error, some bytes are discarded and
 * some bytes are buffered to be converted later.
 * The latter bytes can be put back.
 * It can be observed by
 * Encoding::InvalidByteSequenceError#readagain_bytes and
 * Encoding::Converter#primitive_errinfo.
 *
 *   ec = Encoding::Converter.new("utf-16le", "iso-8859-1")
 *   src = "\x00\xd8\x61\x00"
 *   dst = ""
 *   p ec.primitive_convert(src, dst)   #=> :invalid_byte_sequence
 *   p ec.primitive_errinfo     #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "a\x00"]
 *   p ec.putback               #=> "a\x00"
 *   p ec.putback               #=> ""          # no more bytes to put back
 *
 */
static VALUE
econv_putback(int argc, VALUE *argv, VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    int n;
    int putbackable;
    VALUE str, max;
 
    if (!rb_check_arity(argc, 0, 1) || NIL_P(max = argv[0])) {
        n = rb_econv_putbackable(ec);
    }
    else {
        n = NUM2INT(max);
        putbackable = rb_econv_putbackable(ec);
        if (putbackable < n)
            n = putbackable;
    }
 
    str = rb_str_new(NULL, n);
    rb_econv_putback(ec, (unsigned char *)RSTRING_PTR(str), n);
 
    if (ec->source_encoding) {
        rb_enc_associate(str, ec->source_encoding);
    }
 
    return str;
}
 
/*
 * call-seq:
 *   ec.last_error -> exception or nil
 *
 * Returns an exception object for the last conversion.
 * Returns nil if the last conversion did not produce an error.
 *
 * "error" means that
 * Encoding::InvalidByteSequenceError and Encoding::UndefinedConversionError for
 * Encoding::Converter#convert and
 * :invalid_byte_sequence, :incomplete_input and :undefined_conversion for
 * Encoding::Converter#primitive_convert.
 *
 *  ec = Encoding::Converter.new("utf-8", "iso-8859-1")
 *  p ec.primitive_convert(src="\xf1abcd", dst="")       #=> :invalid_byte_sequence
 *  p ec.last_error      #=> #<Encoding::InvalidByteSequenceError: "\xF1" followed by "a" on UTF-8>
 *  p ec.primitive_convert(src, dst, nil, 1)             #=> :destination_buffer_full
 *  p ec.last_error      #=> nil
 *
 */
static VALUE
econv_last_error(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    VALUE exc;
 
    exc = make_econv_exception(ec);
    if (NIL_P(exc))
        return Qnil;
    return exc;
}
 
/*
 * call-seq:
 *   ec.replacement -> string
 *
 * Returns the replacement string.
 *
 *  ec = Encoding::Converter.new("euc-jp", "us-ascii")
 *  p ec.replacement    #=> "?"
 *
 *  ec = Encoding::Converter.new("euc-jp", "utf-8")
 *  p ec.replacement    #=> "\uFFFD"
 */
static VALUE
econv_get_replacement(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    int ret;
    rb_encoding *enc;
 
    ret = make_replacement(ec);
    if (ret == -1) {
        rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
    }
 
    enc = rb_enc_find(ec->replacement_enc);
    return rb_enc_str_new((const char *)ec->replacement_str, (long)ec->replacement_len, enc);
}
 
/*
 * call-seq:
 *   ec.replacement = string
 *
 * Sets the replacement string.
 *
 *  ec = Encoding::Converter.new("utf-8", "us-ascii", :undef => :replace)
 *  ec.replacement = "<undef>"
 *  p ec.convert("a \u3042 b")      #=> "a <undef> b"
 */
static VALUE
econv_set_replacement(VALUE self, VALUE arg)
{
    rb_econv_t *ec = check_econv(self);
    VALUE string = arg;
    int ret;
    rb_encoding *enc;
 
    StringValue(string);
    enc = rb_enc_get(string);
 
    ret = rb_econv_set_replacement(ec,
            (const unsigned char *)RSTRING_PTR(string),
            RSTRING_LEN(string),
            rb_enc_name(enc));
 
    if (ret == -1) {
        /* xxx: rb_eInvalidByteSequenceError? */
        rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
    }
 
    return arg;
}
 
VALUE
rb_econv_make_exception(rb_econv_t *ec)
{
    return make_econv_exception(ec);
}
 
void
rb_econv_check_error(rb_econv_t *ec)
{
    VALUE exc;
 
    exc = make_econv_exception(ec);
    if (NIL_P(exc))
        return;
    rb_exc_raise(exc);
}
 
/*
 * call-seq:
 *   ecerr.source_encoding_name         -> string
 *
 * Returns the source encoding name as a string.
 */
static VALUE
ecerr_source_encoding_name(VALUE self)
{
    return rb_attr_get(self, id_source_encoding_name);
}
 
/*
 * call-seq:
 *   ecerr.source_encoding              -> encoding
 *
 * Returns the source encoding as an encoding object.
 *
 * Note that the result may not be equal to the source encoding of
 * the encoding converter if the conversion has multiple steps.
 *
 *  ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") # ISO-8859-1 -> UTF-8 -> EUC-JP
 *  begin
 *    ec.convert("\xa0") # NO-BREAK SPACE, which is available in UTF-8 but not in EUC-JP.
 *  rescue Encoding::UndefinedConversionError
 *    p $!.source_encoding              #=> #<Encoding:UTF-8>
 *    p $!.destination_encoding         #=> #<Encoding:EUC-JP>
 *    p $!.source_encoding_name         #=> "UTF-8"
 *    p $!.destination_encoding_name    #=> "EUC-JP"
 *  end
 *
 */
static VALUE
ecerr_source_encoding(VALUE self)
{
    return rb_attr_get(self, id_source_encoding);
}
 
/*
 * call-seq:
 *   ecerr.destination_encoding_name         -> string
 *
 * Returns the destination encoding name as a string.
 */
static VALUE
ecerr_destination_encoding_name(VALUE self)
{
    return rb_attr_get(self, id_destination_encoding_name);
}
 
/*
 * call-seq:
 *   ecerr.destination_encoding         -> string
 *
 * Returns the destination encoding as an encoding object.
 */
static VALUE
ecerr_destination_encoding(VALUE self)
{
    return rb_attr_get(self, id_destination_encoding);
}
 
/*
 * call-seq:
 *   ecerr.error_char         -> string
 *
 * Returns the one-character string which cause Encoding::UndefinedConversionError.
 *
 *  ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP")
 *  begin
 *    ec.convert("\xa0")
 *  rescue Encoding::UndefinedConversionError
 *    puts $!.error_char.dump   #=> "\xC2\xA0"
 *    p $!.error_char.encoding  #=> #<Encoding:UTF-8>
 *  end
 *
 */
static VALUE
ecerr_error_char(VALUE self)
{
    return rb_attr_get(self, id_error_char);
}
 
/*
 * call-seq:
 *   ecerr.error_bytes         -> string
 *
 * Returns the discarded bytes when Encoding::InvalidByteSequenceError occurs.
 *
 *  ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
 *  begin
 *    ec.convert("abc\xA1\xFFdef")
 *  rescue Encoding::InvalidByteSequenceError
 *    p $!      #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "\xFF" on EUC-JP>
 *    puts $!.error_bytes.dump          #=> "\xA1"
 *    puts $!.readagain_bytes.dump      #=> "\xFF"
 *  end
 */
static VALUE
ecerr_error_bytes(VALUE self)
{
    return rb_attr_get(self, id_error_bytes);
}
 
/*
 * call-seq:
 *   ecerr.readagain_bytes         -> string
 *
 * Returns the bytes to be read again when Encoding::InvalidByteSequenceError occurs.
 */
static VALUE
ecerr_readagain_bytes(VALUE self)
{
    return rb_attr_get(self, id_readagain_bytes);
}
 
/*
 * call-seq:
 *   ecerr.incomplete_input?         -> true or false
 *
 * Returns true if the invalid byte sequence error is caused by
 * premature end of string.
 *
 *  ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
 *
 *  begin
 *    ec.convert("abc\xA1z")
 *  rescue Encoding::InvalidByteSequenceError
 *    p $!      #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "z" on EUC-JP>
 *    p $!.incomplete_input?    #=> false
 *  end
 *
 *  begin
 *    ec.convert("abc\xA1")
 *    ec.finish
 *  rescue Encoding::InvalidByteSequenceError
 *    p $!      #=> #<Encoding::InvalidByteSequenceError: incomplete "\xA1" on EUC-JP>
 *    p $!.incomplete_input?    #=> true
 *  end
 */
static VALUE
ecerr_incomplete_input(VALUE self)
{
    return rb_attr_get(self, id_incomplete_input);
}
 
/*
 *  Document-class: Encoding::UndefinedConversionError
 *
 *  Raised by Encoding and String methods when a transcoding operation
 *  fails.
 */
 
/*
 *  Document-class: Encoding::InvalidByteSequenceError
 *
 *  Raised by Encoding and String methods when the string being
 *  transcoded contains a byte invalid for the either the source or
 *  target encoding.
 */
 
/*
 *  Document-class: Encoding::ConverterNotFoundError
 *
 *  Raised by transcoding methods when a named encoding does not
 *  correspond with a known converter.
 */
 
void
Init_transcode(void)
{
    transcoder_table = st_init_strcasetable();
 
    id_destination_encoding = rb_intern_const("destination_encoding");
    id_destination_encoding_name = rb_intern_const("destination_encoding_name");
    id_error_bytes = rb_intern_const("error_bytes");
    id_error_char = rb_intern_const("error_char");
    id_incomplete_input = rb_intern_const("incomplete_input");
    id_readagain_bytes = rb_intern_const("readagain_bytes");
    id_source_encoding = rb_intern_const("source_encoding");
    id_source_encoding_name = rb_intern_const("source_encoding_name");
 
    sym_invalid = ID2SYM(rb_intern_const("invalid"));
    sym_undef = ID2SYM(rb_intern_const("undef"));
    sym_replace = ID2SYM(rb_intern_const("replace"));
    sym_fallback = ID2SYM(rb_intern_const("fallback"));
    sym_xml = ID2SYM(rb_intern_const("xml"));
    sym_text = ID2SYM(rb_intern_const("text"));
    sym_attr = ID2SYM(rb_intern_const("attr"));
 
    sym_invalid_byte_sequence = ID2SYM(rb_intern_const("invalid_byte_sequence"));
    sym_undefined_conversion = ID2SYM(rb_intern_const("undefined_conversion"));
    sym_destination_buffer_full = ID2SYM(rb_intern_const("destination_buffer_full"));
    sym_source_buffer_empty = ID2SYM(rb_intern_const("source_buffer_empty"));
    sym_finished = ID2SYM(rb_intern_const("finished"));
    sym_after_output = ID2SYM(rb_intern_const("after_output"));
    sym_incomplete_input = ID2SYM(rb_intern_const("incomplete_input"));
    sym_universal_newline = ID2SYM(rb_intern_const("universal_newline"));
    sym_crlf_newline = ID2SYM(rb_intern_const("crlf_newline"));
    sym_cr_newline = ID2SYM(rb_intern_const("cr_newline"));
    sym_lf_newline = ID2SYM(rb_intern("lf_newline"));
    sym_partial_input = ID2SYM(rb_intern_const("partial_input"));
 
#ifdef ENABLE_ECONV_NEWLINE_OPTION
    sym_newline = ID2SYM(rb_intern_const("newline"));
    sym_universal = ID2SYM(rb_intern_const("universal"));
    sym_crlf = ID2SYM(rb_intern_const("crlf"));
    sym_cr = ID2SYM(rb_intern_const("cr"));
    sym_lf = ID2SYM(rb_intern_const("lf"));
#endif
 
    InitVM(transcode);
}
 
void
InitVM_transcode(void)
{
    rb_eUndefinedConversionError = rb_define_class_under(rb_cEncoding, "UndefinedConversionError", rb_eEncodingError);
    rb_eInvalidByteSequenceError = rb_define_class_under(rb_cEncoding, "InvalidByteSequenceError", rb_eEncodingError);
    rb_eConverterNotFoundError = rb_define_class_under(rb_cEncoding, "ConverterNotFoundError", rb_eEncodingError);
 
    rb_define_method(rb_cString, "encode", str_encode, -1);
    rb_define_method(rb_cString, "encode!", str_encode_bang, -1);
 
    rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cObject);
    rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate);
    rb_define_singleton_method(rb_cEncodingConverter, "asciicompat_encoding", econv_s_asciicompat_encoding, 1);
    rb_define_singleton_method(rb_cEncodingConverter, "search_convpath", econv_s_search_convpath, -1);
    rb_define_method(rb_cEncodingConverter, "initialize", econv_init, -1);
    rb_define_method(rb_cEncodingConverter, "inspect", econv_inspect, 0);
    rb_define_method(rb_cEncodingConverter, "convpath", econv_convpath, 0);
    rb_define_method(rb_cEncodingConverter, "source_encoding", econv_source_encoding, 0);
    rb_define_method(rb_cEncodingConverter, "destination_encoding", econv_destination_encoding, 0);
    rb_define_method(rb_cEncodingConverter, "primitive_convert", econv_primitive_convert, -1);
    rb_define_method(rb_cEncodingConverter, "convert", econv_convert, 1);
    rb_define_method(rb_cEncodingConverter, "finish", econv_finish, 0);
    rb_define_method(rb_cEncodingConverter, "primitive_errinfo", econv_primitive_errinfo, 0);
    rb_define_method(rb_cEncodingConverter, "insert_output", econv_insert_output, 1);
    rb_define_method(rb_cEncodingConverter, "putback", econv_putback, -1);
    rb_define_method(rb_cEncodingConverter, "last_error", econv_last_error, 0);
    rb_define_method(rb_cEncodingConverter, "replacement", econv_get_replacement, 0);
    rb_define_method(rb_cEncodingConverter, "replacement=", econv_set_replacement, 1);
    rb_define_method(rb_cEncodingConverter, "==", econv_equal, 1);
 
    /*
     *Mask for invalid byte sequences
     */
    rb_define_const(rb_cEncodingConverter, "INVALID_MASK", INT2FIX(ECONV_INVALID_MASK));
 
    /*
     * Replace invalid byte sequences
     */
    rb_define_const(rb_cEncodingConverter, "INVALID_REPLACE", INT2FIX(ECONV_INVALID_REPLACE));
 
    /*
     * Mask for a valid character in the source encoding but no related
     * character(s) in destination encoding.
     */
    rb_define_const(rb_cEncodingConverter, "UNDEF_MASK", INT2FIX(ECONV_UNDEF_MASK));
 
    /*
     * Replace byte sequences that are undefined in the destination encoding.
     */
    rb_define_const(rb_cEncodingConverter, "UNDEF_REPLACE", INT2FIX(ECONV_UNDEF_REPLACE));
 
    /*
     * Replace byte sequences that are undefined in the destination encoding
     * with an XML hexadecimal character reference.  This is valid for XML
     * conversion.
     */
    rb_define_const(rb_cEncodingConverter, "UNDEF_HEX_CHARREF", INT2FIX(ECONV_UNDEF_HEX_CHARREF));
 
    /*
     * Indicates the source may be part of a larger string.  See
     * primitive_convert for an example.
     */
    rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(ECONV_PARTIAL_INPUT));
 
    /*
     * Stop converting after some output is complete but before all of the
     * input was consumed.  See primitive_convert for an example.
     */
    rb_define_const(rb_cEncodingConverter, "AFTER_OUTPUT", INT2FIX(ECONV_AFTER_OUTPUT));
 
    /*
     * Decorator for converting CRLF and CR to LF
     */
    rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECORATOR", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR));
 
    /*
     * Decorator for converting CRLF and CR to LF when writing
     */
    rb_define_const(rb_cEncodingConverter, "LF_NEWLINE_DECORATOR", INT2FIX(ECONV_LF_NEWLINE_DECORATOR));
 
    /*
     * Decorator for converting LF to CRLF
     */
    rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_DECORATOR", INT2FIX(ECONV_CRLF_NEWLINE_DECORATOR));
 
    /*
     * Decorator for converting LF to CR
     */
    rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_DECORATOR", INT2FIX(ECONV_CR_NEWLINE_DECORATOR));
 
    /*
     * Escape as XML CharData
     */
    rb_define_const(rb_cEncodingConverter, "XML_TEXT_DECORATOR", INT2FIX(ECONV_XML_TEXT_DECORATOR));
 
    /*
     * Escape as XML AttValue
     */
    rb_define_const(rb_cEncodingConverter, "XML_ATTR_CONTENT_DECORATOR", INT2FIX(ECONV_XML_ATTR_CONTENT_DECORATOR));
 
    /*
     * Escape as XML AttValue
     */
    rb_define_const(rb_cEncodingConverter, "XML_ATTR_QUOTE_DECORATOR", INT2FIX(ECONV_XML_ATTR_QUOTE_DECORATOR));
 
    rb_define_method(rb_eUndefinedConversionError, "source_encoding_name", ecerr_source_encoding_name, 0);
    rb_define_method(rb_eUndefinedConversionError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
    rb_define_method(rb_eUndefinedConversionError, "source_encoding", ecerr_source_encoding, 0);
    rb_define_method(rb_eUndefinedConversionError, "destination_encoding", ecerr_destination_encoding, 0);
    rb_define_method(rb_eUndefinedConversionError, "error_char", ecerr_error_char, 0);
 
    rb_define_method(rb_eInvalidByteSequenceError, "source_encoding_name", ecerr_source_encoding_name, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "source_encoding", ecerr_source_encoding, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding", ecerr_destination_encoding, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "error_bytes", ecerr_error_bytes, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "readagain_bytes", ecerr_readagain_bytes, 0);
    rb_define_method(rb_eInvalidByteSequenceError, "incomplete_input?", ecerr_incomplete_input, 0);
 
    Init_newline();
}