12#include "ruby/internal/config.h"
17#include "internal/array.h"
18#include "internal/inits.h"
19#include "internal/gc.h"
20#include "internal/object.h"
21#include "internal/string.h"
22#include "internal/transcode.h"
23#include "internal/encoding.h"
27#include "transcode_data.h"
30#define ENABLE_ECONV_NEWLINE_OPTION 1
33static VALUE rb_eUndefinedConversionError;
34static VALUE rb_eInvalidByteSequenceError;
35static VALUE rb_eConverterNotFoundError;
37VALUE rb_cEncodingConverter;
39static ID id_destination_encoding;
40static ID id_destination_encoding_name;
41static ID id_error_bytes;
42static ID id_error_char;
43static ID id_incomplete_input;
44static ID id_readagain_bytes;
45static ID id_source_encoding;
46static ID id_source_encoding_name;
48static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
49static VALUE sym_xml, sym_text, sym_attr;
50static VALUE sym_universal_newline;
51static VALUE sym_crlf_newline;
52static VALUE sym_cr_newline;
53static VALUE sym_lf_newline;
54#ifdef ENABLE_ECONV_NEWLINE_OPTION
55static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
57static VALUE sym_partial_input;
59static VALUE sym_invalid_byte_sequence;
60static VALUE sym_undefined_conversion;
61static VALUE sym_destination_buffer_full;
62static VALUE sym_source_buffer_empty;
63static VALUE sym_finished;
64static VALUE sym_after_output;
65static VALUE sym_incomplete_input;
68allocate_converted_string(
const char *sname,
const char *dname,
69 const unsigned char *str,
size_t len,
70 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
71 size_t *dst_len_ptr,
size_t *dst_bufsize_ptr);
81 unsigned int next_table;
83 unsigned char next_byte;
84 unsigned int output_index;
86 ssize_t recognized_len;
87 ssize_t readagain_len;
102 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
103 double dummy_for_alignment;
106#define TRANSCODING_READBUF(tc) \
107 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
108 (tc)->readbuf.ary : \
110#define TRANSCODING_WRITEBUF(tc) \
111 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
112 (tc)->writebuf.ary : \
114#define TRANSCODING_WRITEBUF_SIZE(tc) \
115 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
116 sizeof((tc)->writebuf.ary) : \
117 (size_t)(tc)->transcoder->max_output)
118#define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
119#define TRANSCODING_STATE(tc) \
120 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
126 unsigned char *out_buf_start;
127 unsigned char *out_data_start;
128 unsigned char *out_data_end;
129 unsigned char *out_buf_end;
137 const char *source_encoding_name;
138 const char *destination_encoding_name;
140 const unsigned char *replacement_str;
141 size_t replacement_len;
142 size_t replacement_bufsize;
143 const char *replacement_enc;
145 unsigned char *in_buf_start;
146 unsigned char *in_data_start;
147 unsigned char *in_data_end;
148 unsigned char *in_buf_end;
150 int replacement_allocated;
160 const char *source_encoding;
161 const char *destination_encoding;
162 const unsigned char *error_bytes_start;
163 size_t error_bytes_len;
164 size_t readagain_len;
177#define DECORATOR_P(sname, dname) (*(sname) == '\0')
189free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
196free_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
198 st_foreach((
void *)val, free_inner_transcode_i, 0);
199 st_free_table((
void *)val);
204rb_free_transcoder_table(
void)
206 st_foreach(transcoder_table, free_transcode_i, 0);
207 st_free_table(transcoder_table);
211make_transcoder_entry(
const char *sname,
const char *dname)
217 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
218 val = (st_data_t)st_init_strcasetable();
219 st_add_direct(transcoder_table, (st_data_t)sname, val);
222 if (!st_lookup(table2, (st_data_t)dname, &val)) {
224 entry->sname = sname;
225 entry->dname = dname;
227 entry->transcoder = NULL;
228 val = (st_data_t)entry;
229 st_add_direct(table2, (st_data_t)dname, val);
236get_transcoder_entry(
const char *sname,
const char *dname)
241 if (st_lookup(transcoder_table, (st_data_t)sname, &val)) {
243 if (!st_lookup(table2, (st_data_t)dname, &val)) {
254 const char *
const sname =
tr->src_encoding;
255 const char *
const dname =
tr->dst_encoding;
260 entry = make_transcoder_entry(sname, dname);
261 if (entry->transcoder) {
262 rb_raise(rb_eArgError,
"transcoder from %s to %s has been already registered",
265 entry->transcoder =
tr;
270declare_transcoder(
const char *sname,
const char *dname,
const char *lib)
274 entry = make_transcoder_entry(sname, dname);
278static const char transcoder_lib_prefix[] =
"enc/trans/";
281rb_declare_transcoder(
const char *enc1,
const char *enc2,
const char *lib)
284 rb_raise(rb_eArgError,
"invalid library name - (null)");
286 declare_transcoder(enc1, enc2, lib);
289#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
300 const char *base_enc;
304transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
306 const char *dname = (
const char *)key;
310 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
317 *bfs->queue_last_ptr = q;
318 bfs->queue_last_ptr = &q->next;
320 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
325transcode_search_path(
const char *sname,
const char *dname,
326 void (*callback)(
const char *sname,
const char *dname,
int depth,
void *arg),
337 if (encoding_equal(sname, dname))
343 bfs.queue_last_ptr = &q->next;
346 bfs.visited = st_init_strcasetable();
347 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
354 bfs.queue_last_ptr = &bfs.queue;
357 lookup_res = st_lookup(transcoder_table, (st_data_t)q->enc, &val);
364 if (st_lookup(table2, (st_data_t)dname, &val)) {
365 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
371 bfs.base_enc = q->enc;
372 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
386 const char *enc = dname;
390 st_lookup(bfs.visited, (st_data_t)enc, &val);
394 enc = (
const char *)val;
399 st_lookup(bfs.visited, (st_data_t)enc, &val);
402 callback((
const char *)val, enc, --depth, arg);
403 enc = (
const char *)val;
407 st_free_table(bfs.visited);
412int rb_require_internal_silent(
VALUE fname);
417 ASSERT_vm_unlocking();
418 if (entry->transcoder)
419 return entry->transcoder;
422 const char *
const lib = entry->lib;
423 const size_t len = strlen(lib);
424 const size_t total_len =
sizeof(transcoder_lib_prefix) - 1 +
len;
426 char *
const path = RSTRING_PTR(fn);
428 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
429 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib,
len);
432 rb_require_internal_silent(fn);
435 if (entry->transcoder)
436 return entry->transcoder;
442get_replacement_character(
const char *encname,
size_t *len_ret,
const char **repl_encname_ptr)
444 if (encoding_equal(encname,
"UTF-8")) {
446 *repl_encname_ptr =
"UTF-8";
447 return "\xEF\xBF\xBD";
451 *repl_encname_ptr =
"US-ASCII";
460static const unsigned char *
462 const unsigned char *in_start,
463 const unsigned char *inchar_start,
464 const unsigned char *in_p,
465 size_t *char_len_ptr)
467 const unsigned char *ptr;
468 if (inchar_start - in_start < tc->recognized_len) {
469 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
470 inchar_start,
unsigned char, in_p - inchar_start);
471 ptr = TRANSCODING_READBUF(tc);
474 ptr = inchar_start - tc->recognized_len;
476 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
481transcode_restartable0(
const unsigned char **in_pos,
unsigned char **out_pos,
482 const unsigned char *in_stop,
unsigned char *out_stop,
487 int unitlen =
tr->input_unit_length;
488 ssize_t readagain_len = 0;
490 const unsigned char *inchar_start;
491 const unsigned char *in_p;
493 unsigned char *out_p;
495 in_p = inchar_start = *in_pos;
499#define SUSPEND(ret, num) \
501 tc->resume_position = (num); \
502 if (0 < in_p - inchar_start) \
503 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
504 inchar_start, unsigned char, in_p - inchar_start); \
507 tc->recognized_len += in_p - inchar_start; \
508 if (readagain_len) { \
509 tc->recognized_len -= readagain_len; \
510 tc->readagain_len = readagain_len; \
513 resume_label ## num:; \
515#define SUSPEND_OBUF(num) \
517 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
520#define SUSPEND_AFTER_OUTPUT(num) \
521 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
522 SUSPEND(econv_after_output, num); \
525#define next_table (tc->next_table)
526#define next_info (tc->next_info)
527#define next_byte (tc->next_byte)
528#define writebuf_len (tc->writebuf_len)
529#define writebuf_off (tc->writebuf_off)
531 switch (tc->resume_position) {
533 case 1:
goto resume_label1;
534 case 2:
goto resume_label2;
535 case 3:
goto resume_label3;
536 case 4:
goto resume_label4;
537 case 5:
goto resume_label5;
538 case 6:
goto resume_label6;
539 case 7:
goto resume_label7;
540 case 8:
goto resume_label8;
541 case 9:
goto resume_label9;
542 case 10:
goto resume_label10;
543 case 11:
goto resume_label11;
544 case 12:
goto resume_label12;
545 case 13:
goto resume_label13;
546 case 14:
goto resume_label14;
547 case 15:
goto resume_label15;
548 case 16:
goto resume_label16;
549 case 17:
goto resume_label17;
550 case 18:
goto resume_label18;
551 case 19:
goto resume_label19;
552 case 20:
goto resume_label20;
553 case 21:
goto resume_label21;
554 case 22:
goto resume_label22;
555 case 23:
goto resume_label23;
556 case 24:
goto resume_label24;
557 case 25:
goto resume_label25;
558 case 26:
goto resume_label26;
559 case 27:
goto resume_label27;
560 case 28:
goto resume_label28;
561 case 29:
goto resume_label29;
562 case 30:
goto resume_label30;
563 case 31:
goto resume_label31;
564 case 32:
goto resume_label32;
565 case 33:
goto resume_label33;
566 case 34:
goto resume_label34;
571 tc->recognized_len = 0;
572 next_table =
tr->conv_tree_start;
574 SUSPEND_AFTER_OUTPUT(24);
576 if (in_stop <= in_p) {
583#define BYTE_ADDR(index) (tr->byte_array + (index))
584#define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
585#define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
586#define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
587#define BL_MIN_BYTE (BL_BASE[0])
588#define BL_MAX_BYTE (BL_BASE[1])
589#define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
590#define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
592 next_byte = (
unsigned char)*in_p++;
594 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
597 next_info = (
VALUE)BL_ACTION(next_byte);
600 switch (next_info & 0x1F) {
603 const unsigned char *p = inchar_start;
606 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (
unsigned char)*p++;
608 writebuf_len = writebuf_off;
610 while (writebuf_off < writebuf_len) {
612 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
616 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
617 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
618 SUSPEND_AFTER_OUTPUT(25);
619 while (in_p >= in_stop) {
624 next_byte = (
unsigned char)*in_p++;
625 next_table = (
unsigned int)next_info;
630 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
633 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
634 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
637 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
638 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
639 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
642 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
643 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
644 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
645 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
648 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
649 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
650 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
651 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
654 tc->output_index = 0;
655 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
656 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
661 next_info = (
VALUE)(*
tr->func_ii)(TRANSCODING_STATE(tc), next_info);
665 const unsigned char *char_start;
667 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
668 next_info = (
VALUE)(*
tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
673 if (
tr->max_output <= out_stop - out_p)
674 out_p +=
tr->func_io(TRANSCODING_STATE(tc),
675 next_info, out_p, out_stop - out_p);
677 writebuf_len =
tr->func_io(TRANSCODING_STATE(tc),
679 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
681 while (writebuf_off < writebuf_len) {
683 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
689 const unsigned char *char_start;
692 if (
tr->max_output <= out_stop - out_p) {
693 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
694 out_p +=
tr->func_so(TRANSCODING_STATE(tc),
695 char_start, (
size_t)char_len,
696 out_p, out_stop - out_p);
699 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
700 writebuf_len =
tr->func_so(TRANSCODING_STATE(tc),
701 char_start, (
size_t)char_len,
702 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
704 while (writebuf_off < writebuf_len) {
706 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
713 const unsigned char *char_start;
716 if (
tr->max_output <= out_stop - out_p) {
717 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
718 out_p +=
tr->func_sio(TRANSCODING_STATE(tc),
719 char_start, (
size_t)char_len, next_info,
720 out_p, out_stop - out_p);
723 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
724 writebuf_len =
tr->func_sio(TRANSCODING_STATE(tc),
725 char_start, (
size_t)char_len, next_info,
726 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
728 while (writebuf_off < writebuf_len) {
730 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
736 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
737 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
738 SUSPEND_AFTER_OUTPUT(26);
739 while ((opt &
ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
743 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
747 in_p = inchar_start + (unitlen - tc->recognized_len);
753 invalid_len = tc->recognized_len + (in_p - inchar_start);
754 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
755 readagain_len = invalid_len - discard_len;
779 if (
tr->finish_func) {
781 if (
tr->max_output <= out_stop - out_p) {
782 out_p +=
tr->finish_func(TRANSCODING_STATE(tc),
783 out_p, out_stop - out_p);
786 writebuf_len =
tr->finish_func(TRANSCODING_STATE(tc),
787 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
789 while (writebuf_off < writebuf_len) {
791 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
806transcode_restartable(
const unsigned char **in_pos,
unsigned char **out_pos,
807 const unsigned char *in_stop,
unsigned char *out_stop,
811 if (tc->readagain_len) {
812 unsigned char *readagain_buf =
ALLOCA_N(
unsigned char, tc->readagain_len);
813 const unsigned char *readagain_pos = readagain_buf;
814 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
817 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
818 unsigned char, tc->readagain_len);
819 tc->readagain_len = 0;
820 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|
ECONV_PARTIAL_INPUT);
822 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
823 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
824 tc->readagain_len += readagain_stop - readagain_pos;
828 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
839 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
841 if (
tr->state_init_func) {
842 (
tr->state_init_func)(TRANSCODING_STATE(tc));
844 tc->resume_position = 0;
845 tc->recognized_len = 0;
846 tc->readagain_len = 0;
847 tc->writebuf_len = 0;
848 tc->writebuf_off = 0;
849 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
850 tc->readbuf.ptr =
xmalloc(
tr->max_input);
852 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
853 tc->writebuf.ptr =
xmalloc(
tr->max_output);
860 const unsigned char **input_ptr,
const unsigned char *input_stop,
861 unsigned char **output_ptr,
unsigned char *output_stop,
864 return transcode_restartable(
865 input_ptr, output_ptr,
866 input_stop, output_stop,
874 if (
tr->state_fini_func) {
875 (
tr->state_fini_func)(TRANSCODING_STATE(tc));
877 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
878 ruby_xfree_sized(tc->state.ptr,
tr->state_size);
879 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input)
880 ruby_xfree_sized(tc->readbuf.ptr,
tr->max_input);
881 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output)
882 ruby_xfree_sized(tc->writebuf.ptr,
tr->max_output);
892 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
893 size +=
tr->state_size;
895 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
896 size +=
tr->max_input;
898 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
899 size +=
tr->max_output;
905rb_econv_alloc(
int n_hint)
914 ec->source_encoding_name = NULL;
915 ec->destination_encoding_name = NULL;
917 ec->replacement_str = NULL;
918 ec->replacement_len = 0;
919 ec->replacement_bufsize = 0;
920 ec->replacement_enc = NULL;
921 ec->replacement_allocated = 0;
922 ec->in_buf_start = NULL;
923 ec->in_data_start = NULL;
924 ec->in_data_end = NULL;
925 ec->in_buf_end = NULL;
926 ec->num_allocated = n_hint;
929 ec->num_finished = 0;
932 ec->last_error.error_tc = NULL;
933 ec->last_error.source_encoding = NULL;
934 ec->last_error.destination_encoding = NULL;
935 ec->last_error.error_bytes_start = NULL;
936 ec->last_error.error_bytes_len = 0;
937 ec->last_error.readagain_len = 0;
938 ec->source_encoding = NULL;
939 ec->destination_encoding = NULL;
950 if (ec->num_trans == ec->num_allocated) {
951 n = ec->num_allocated * 2;
953 ec->num_allocated = n;
960 ec->elems[i].tc = rb_transcoding_open_by_transcoder(
tr, 0);
961 ec->elems[i].out_buf_start = p;
962 ec->elems[i].out_buf_end = p + bufsize;
963 ec->elems[i].out_data_start = p;
964 ec->elems[i].out_data_end = p;
969 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
970 for (j = ec->num_trans-1; i <= j; j--) {
973 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
988 for (i = 0; i < n; i++) {
990 tr = load_transcoder_entry(entries[i]);
995 ec = rb_econv_alloc(n);
997 for (i = 0; i < n; i++) {
999 ret = rb_econv_add_transcoder_at(ec,
tr, ec->num_trans);
1014trans_open_i(
const char *sname,
const char *dname,
int depth,
void *arg)
1018 if (!toarg->entries) {
1021 toarg->entries[depth] = get_transcoder_entry(sname, dname);
1025rb_econv_open0(
const char *sname,
const char *dname,
int ecflags)
1032 if (*sname) rb_enc_find_index(sname);
1033 if (*dname) rb_enc_find_index(dname);
1035 if (*sname ==
'\0' && *dname ==
'\0') {
1042 num_trans = transcode_search_path(sname, dname, trans_open_i, (
void *)&toarg);
1043 entries = toarg.entries;
1044 if (num_trans < 0) {
1045 SIZED_FREE_N(entries, num_trans);
1048 sname = entries[0]->sname;
1049 dname = entries[num_trans-1]->dname;
1052 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1053 SIZED_FREE_N(entries, num_trans);
1057 ec->flags = ecflags;
1058 ec->source_encoding_name = sname;
1059 ec->destination_encoding_name = dname;
1064#define MAX_ECFLAGS_DECORATORS 32
1067decorator_names(
int ecflags,
const char **decorators_ret)
1089 decorators_ret[num_decorators++] =
"xml_text_escape";
1091 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1093 decorators_ret[num_decorators++] =
"xml_attr_quote";
1096 decorators_ret[num_decorators++] =
"crlf_newline";
1098 decorators_ret[num_decorators++] =
"cr_newline";
1100 decorators_ret[num_decorators++] =
"lf_newline";
1102 decorators_ret[num_decorators++] =
"universal_newline";
1104 return num_decorators;
1112 const char *decorators[MAX_ECFLAGS_DECORATORS];
1115 num_decorators = decorator_names(ecflags, decorators);
1116 if (num_decorators == -1)
1121 for (i = 0; i < num_decorators; i++) {
1131 ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1138 const unsigned char **input_ptr,
const unsigned char *input_stop,
1139 unsigned char **output_ptr,
unsigned char *output_stop,
1146 const unsigned char **ipp, *is, *iold;
1147 unsigned char **opp, *os, *oold;
1153 for (i = start; i < ec->num_trans; i++) {
1162 ipp = (
const unsigned char **)&prev_te->out_data_start;
1163 is = prev_te->out_data_end;
1166 if (i == ec->num_trans-1) {
1171 if (te->out_buf_start != te->out_data_start) {
1172 ssize_t
len = te->out_data_end - te->out_data_start;
1173 ssize_t
off = te->out_data_start - te->out_buf_start;
1174 MEMMOVE(te->out_buf_start, te->out_data_start,
unsigned char,
len);
1175 te->out_data_start = te->out_buf_start;
1176 te->out_data_end -=
off;
1178 opp = &te->out_data_end;
1179 os = te->out_buf_end;
1183 if (ec->num_finished != i)
1187 flags &= ~ECONV_AFTER_OUTPUT;
1190 f &= ~ECONV_AFTER_OUTPUT;
1193 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1194 if (iold != *ipp || oold != *opp)
1209 ec->num_finished = i+1;
1219 const unsigned char **input_ptr,
const unsigned char *input_stop,
1220 unsigned char **output_ptr,
unsigned char *output_stop,
1222 int *result_position_ptr)
1225 int needreport_index;
1228 unsigned char empty_buf;
1229 unsigned char *empty_ptr = &empty_buf;
1232 input_ptr = (
const unsigned char **)&empty_ptr;
1233 input_stop = empty_ptr;
1237 output_ptr = &empty_ptr;
1238 output_stop = empty_ptr;
1244 for (i = ec->num_trans-1; 0 <= i; i--) {
1245 switch (ec->elems[i].last_result) {
1252 goto found_needreport;
1259 rb_bug(
"unexpected transcode last result");
1269 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1271 result_position_ptr);
1283 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1284 sweep_start = needreport_index + 1;
1285 }
while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1287 for (i = ec->num_trans-1; 0 <= i; i--) {
1296 if (result_position_ptr)
1297 *result_position_ptr = i;
1301 if (result_position_ptr)
1302 *result_position_ptr = -1;
1308 const unsigned char **input_ptr,
const unsigned char *input_stop,
1309 unsigned char **output_ptr,
unsigned char *output_stop,
1313 int result_position;
1316 memset(&ec->last_error, 0,
sizeof(ec->last_error));
1318 if (ec->num_trans == 0) {
1320 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1321 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1322 len = output_stop - *output_ptr;
1323 memcpy(*output_ptr, ec->in_data_start,
len);
1324 *output_ptr = output_stop;
1325 ec->in_data_start +=
len;
1329 len = ec->in_data_end - ec->in_data_start;
1330 memcpy(*output_ptr, ec->in_data_start,
len);
1332 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1338 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1339 len = output_stop - *output_ptr;
1342 len = input_stop - *input_ptr;
1345 *(*output_ptr)++ = *(*input_ptr)++;
1349 memcpy(*output_ptr, *input_ptr,
len);
1352 if (*input_ptr != input_stop)
1361 if (ec->elems[ec->num_trans-1].out_data_start) {
1362 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1363 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1364 if (data_start != data_end) {
1366 if (output_stop - *output_ptr < data_end - data_start) {
1367 len = output_stop - *output_ptr;
1368 memcpy(*output_ptr, data_start,
len);
1369 *output_ptr = output_stop;
1370 ec->elems[ec->num_trans-1].out_data_start +=
len;
1374 len = data_end - data_start;
1375 memcpy(*output_ptr, data_start,
len);
1377 ec->elems[ec->num_trans-1].out_data_start =
1378 ec->elems[ec->num_trans-1].out_data_end =
1379 ec->elems[ec->num_trans-1].out_buf_start;
1384 if (ec->in_buf_start &&
1385 ec->in_data_start != ec->in_data_end) {
1386 res = rb_trans_conv(ec, (
const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1394 *input_ptr != input_stop) {
1395 input_stop = *input_ptr;
1396 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1401 ec->num_trans == 1) {
1402 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1407 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1412 ec->last_error.result = res;
1417 ec->last_error.error_tc = error_tc;
1418 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1419 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1420 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1421 ec->last_error.error_bytes_len = error_tc->recognized_len;
1422 ec->last_error.readagain_len = error_tc->readagain_len;
1428static int output_replacement_character(
rb_econv_t *ec);
1434 unsigned char utfbuf[1024];
1435 const unsigned char *utf;
1436 size_t utf_len, utf_bufsize;
1437 int utf_allocated = 0;
1438 char charef_buf[16];
1439 const unsigned char *p;
1441 if (encoding_equal(ec->last_error.source_encoding,
"UTF-32BE")) {
1442 utf = ec->last_error.error_bytes_start;
1443 utf_len = ec->last_error.error_bytes_len;
1446 utf = allocate_converted_string(ec->last_error.source_encoding,
"UTF-32BE",
1447 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1448 utfbuf,
sizeof(utfbuf),
1449 &utf_len, &utf_bufsize);
1452 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1456 if (utf_len % 4 != 0)
1460 while (4 <= utf_len) {
1466 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1477 ruby_xfree_sized((
void *)utf, utf_bufsize);
1482 ruby_xfree_sized((
void *)utf, utf_bufsize);
1488 const unsigned char **input_ptr,
const unsigned char *input_stop,
1489 unsigned char **output_ptr,
unsigned char *output_stop,
1494 unsigned char empty_buf;
1495 unsigned char *empty_ptr = &empty_buf;
1500 input_ptr = (
const unsigned char **)&empty_ptr;
1501 input_stop = empty_ptr;
1505 output_ptr = &empty_ptr;
1506 output_stop = empty_ptr;
1510 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1518 if (output_replacement_character(ec) == 0)
1529 if (output_replacement_character(ec) == 0)
1534 if (output_hex_charref(ec) == 0)
1552 tr = tc->transcoder;
1554 if (
tr->asciicompat_type == asciicompat_encoder)
1555 return tr->src_encoding;
1556 return tr->dst_encoding;
1559static unsigned char *
1560allocate_converted_string(
const char *sname,
const char *dname,
1561 const unsigned char *str,
size_t len,
1562 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1563 size_t *dst_len_ptr,
size_t *dst_bufsize_ptr)
1565 unsigned char *dst_str;
1572 const unsigned char *sp;
1576 dst_bufsize = caller_dst_bufsize;
1586 dst_str = caller_dst_buf;
1588 dst_str =
xmalloc(dst_bufsize);
1591 dp = dst_str+dst_len;
1593 dst_len = dp - dst_str;
1595 if (SIZE_MAX/2 < dst_bufsize) {
1599 if (dst_str == caller_dst_buf) {
1602 memcpy(tmp, dst_str, dst_bufsize/2);
1606 dst_str = ruby_xrealloc_sized(dst_str, dst_bufsize, dst_bufsize / 2);
1608 dp = dst_str+dst_len;
1610 dst_len = dp - dst_str;
1616 *dst_len_ptr = dst_len;
1617 *dst_bufsize_ptr = dst_bufsize;
1621 if (dst_str != caller_dst_buf)
1622 ruby_xfree_sized(dst_str, dst_bufsize);
1630 const unsigned char *str,
size_t len,
const char *str_encoding)
1633 unsigned char insert_buf[4096];
1634 const unsigned char *insert_str = NULL;
1635 size_t insert_len, insert_bufsize;
1637 int last_trans_index;
1640 unsigned char **buf_start_p;
1641 unsigned char **data_start_p;
1642 unsigned char **data_end_p;
1643 unsigned char **buf_end_p;
1652 if (encoding_equal(insert_encoding, str_encoding)) {
1657 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1658 str,
len, insert_buf,
sizeof(insert_buf), &insert_len, &insert_bufsize);
1659 if (insert_str == NULL)
1665 last_trans_index = ec->num_trans-1;
1666 if (ec->num_trans == 0) {
1668 buf_start_p = &ec->in_buf_start;
1669 data_start_p = &ec->in_data_start;
1670 data_end_p = &ec->in_data_end;
1671 buf_end_p = &ec->in_buf_end;
1673 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1674 tc = ec->elems[last_trans_index].tc;
1675 need += tc->readagain_len;
1676 if (need < insert_len)
1678 if (last_trans_index == 0) {
1679 buf_start_p = &ec->in_buf_start;
1680 data_start_p = &ec->in_data_start;
1681 data_end_p = &ec->in_data_end;
1682 buf_end_p = &ec->in_buf_end;
1686 buf_start_p = &ee->out_buf_start;
1687 data_start_p = &ee->out_data_start;
1688 data_end_p = &ee->out_data_end;
1689 buf_end_p = &ee->out_buf_end;
1694 buf_start_p = &ee->out_buf_start;
1695 data_start_p = &ee->out_data_start;
1696 data_end_p = &ee->out_data_end;
1697 buf_end_p = &ee->out_buf_end;
1698 tc = ec->elems[last_trans_index].tc;
1701 if (*buf_start_p == NULL) {
1702 unsigned char *buf =
xmalloc(need);
1704 *data_start_p = buf;
1706 *buf_end_p = buf+need;
1708 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1709 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1710 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1711 *data_start_p = *buf_start_p;
1712 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1714 size_t s = (*data_end_p - *buf_start_p) + need;
1717 buf = ruby_xrealloc_sized(*buf_start_p, s, *buf_end_p - *buf_start_p);
1718 *data_start_p = buf;
1719 *data_end_p = buf + (*data_end_p - *buf_start_p);
1721 *buf_end_p = buf + s;
1725 memcpy(*data_end_p, insert_str, insert_len);
1726 *data_end_p += insert_len;
1727 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1728 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1729 *data_end_p += tc->readagain_len;
1730 tc->readagain_len = 0;
1733 if (insert_str != str && insert_str != insert_buf)
1734 ruby_xfree_sized((
void *)insert_str, insert_bufsize);
1738 if (insert_str != str && insert_str != insert_buf)
1739 ruby_xfree_sized((
void *)insert_str, insert_bufsize);
1748 if (ec->replacement_allocated) {
1749 SIZED_FREE_N((
char *)ec->replacement_str, ec->replacement_bufsize);
1751 for (i = 0; i < ec->num_trans; i++) {
1752 rb_transcoding_close(ec->elems[i].tc);
1753 ruby_xfree_sized(ec->elems[i].out_buf_start, ec->elems[i].out_buf_end - ec->elems[i].out_buf_start);
1755 SIZED_FREE_N(ec->in_buf_start, ec->in_buf_end - ec->in_buf_start);
1756 SIZED_FREE_N(ec->elems, ec->num_allocated);
1766 if (ec->replacement_allocated) {
1767 size += ec->replacement_len;
1769 for (i = 0; i < ec->num_trans; i++) {
1770 size += rb_transcoding_memsize(ec->elems[i].tc);
1772 if (ec->elems[i].out_buf_start) {
1773 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1776 size += ec->in_buf_end - ec->in_buf_start;
1785 if (ec->num_trans == 0)
1787#if SIZEOF_SIZE_T > SIZEOF_INT
1788 if (ec->elems[0].tc->readagain_len > INT_MAX)
return INT_MAX;
1790 return (
int)ec->elems[0].tc->readagain_len;
1797 if (ec->num_trans == 0 || n == 0)
1799 tc = ec->elems[0].tc;
1800 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1801 tc->readagain_len -= n;
1805 const char *ascii_compat_name;
1806 const char *ascii_incompat_name;
1810asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1816 if (DECORATOR_P(entry->sname, entry->dname))
1818 tr = load_transcoder_entry(entry);
1819 if (
tr &&
tr->asciicompat_type == asciicompat_decoder) {
1820 data->ascii_compat_name =
tr->dst_encoding;
1834 RB_VM_LOCK_ENTER_LEV(&lev);
1836 if (st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v)) {
1845 if (table2->num_entries == 1) {
1846 data.ascii_incompat_name = ascii_incompat_name;
1847 data.ascii_compat_name = NULL;
1848 if (rb_multi_ractor_p()) {
1853 st_table *dup_table2 = st_copy(table2);
1854 RB_VM_LOCK_LEAVE_LEV(&lev);
1855 st_foreach(dup_table2, asciicompat_encoding_i, (st_data_t)&data);
1856 st_free_table(dup_table2);
1857 RB_VM_LOCK_ENTER_LEV(&lev);
1860 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1866 RB_VM_LOCK_LEAVE_LEV(&lev);
1868 return data.ascii_compat_name;
1880 unsigned const char *sp, *se;
1881 unsigned char *ds, *dp, *de;
1890 rb_enc_associate(dst, dst_enc);
1895 dst_enc = rb_enc_get(dst);
1896 coderange = rb_enc_str_coderange(dst);
1900 max_output = ec->last_tc->transcoder->max_output;
1910 rb_raise(rb_eArgError,
"too long string");
1913 sp = (
const unsigned char *)ss;
1915 ds = (
unsigned char *)RSTRING_PTR(dst);
1919 switch (coderange) {
1922 cr = (int)coderange;
1931 len -= (
const char *)sp - ss;
1932 ss = (
const char *)sp;
1968rb_econv_add_converter(
rb_econv_t *ec,
const char *sname,
const char *dname,
int n)
1973 if (ec->started != 0)
1976 entry = get_transcoder_entry(sname, dname);
1978 tr = load_transcoder_entry(entry);
1981 return tr ? rb_econv_add_transcoder_at(ec,
tr, n) : -1;
1985rb_econv_decorate_at(
rb_econv_t *ec,
const char *decorator_name,
int n)
1987 return rb_econv_add_converter(ec,
"", decorator_name, n);
1995 if (ec->num_trans == 0)
1996 return rb_econv_decorate_at(ec, decorator_name, 0);
1998 tr = ec->elems[0].tc->transcoder;
2000 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
2001 tr->asciicompat_type == asciicompat_decoder)
2002 return rb_econv_decorate_at(ec, decorator_name, 1);
2004 return rb_econv_decorate_at(ec, decorator_name, 0);
2012 if (ec->num_trans == 0)
2013 return rb_econv_decorate_at(ec, decorator_name, 0);
2015 tr = ec->elems[ec->num_trans-1].tc->transcoder;
2017 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
2018 tr->asciicompat_type == asciicompat_encoder)
2019 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
2021 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
2027 const char *dname = 0;
2031 dname =
"universal_newline";
2034 dname =
"crlf_newline";
2037 dname =
"cr_newline";
2040 dname =
"lf_newline";
2045 const rb_transcoder *transcoder = get_transcoder_entry(
"", dname)->transcoder;
2046 int num_trans = ec->num_trans;
2049 for (i=0; i < num_trans; i++) {
2050 if (transcoder == ec->elems[i].tc->transcoder) {
2051 rb_transcoding_close(ec->elems[i].tc);
2052 ruby_xfree_sized(ec->elems[i].out_buf_start, ec->elems[i].out_buf_end - ec->elems[i].out_buf_start);
2056 ec->elems[j++] = ec->elems[i];
2060 ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2064econv_description(
const char *sname,
const char *dname,
int ecflags,
VALUE mesg)
2066 int has_description = 0;
2071 if (*sname !=
'\0' || *dname !=
'\0') {
2074 else if (*dname ==
'\0')
2077 rb_str_catf(mesg,
"%s to %s", sname, dname);
2078 has_description = 1;
2085 const char *pre =
"";
2086 if (has_description)
2116 has_description = 1;
2118 if (!has_description) {
2130 econv_description(sname, dname, ecflags, mesg);
2132 exc =
rb_exc_new3(rb_eConverterNotFoundError, mesg);
2142 const char *err = (
const char *)ec->last_error.error_bytes_start;
2143 size_t error_len = ec->last_error.error_bytes_len;
2146 size_t readagain_len = ec->last_error.readagain_len;
2150 mesg = rb_sprintf(
"incomplete %s on %s",
2152 ec->last_error.source_encoding);
2154 else if (readagain_len) {
2155 bytes2 =
rb_str_new(err+error_len, readagain_len);
2157 mesg = rb_sprintf(
"%s followed by %s on %s",
2160 ec->last_error.source_encoding);
2163 mesg = rb_sprintf(
"%s on %s",
2165 ec->last_error.source_encoding);
2168 exc =
rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2175 VALUE bytes =
rb_str_new((
const char *)ec->last_error.error_bytes_start,
2176 ec->last_error.error_bytes_len);
2179 if (strcmp(ec->last_error.source_encoding,
"UTF-8") == 0) {
2181 const char *start, *end;
2183 start = (
const char *)ec->last_error.error_bytes_start;
2184 end = start + ec->last_error.error_bytes_len;
2185 n = rb_enc_precise_mbclen(start, end, utf8);
2188 unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
2189 dumped = rb_sprintf(
"U+%04X", cc);
2194 if (strcmp(ec->last_error.source_encoding,
2195 ec->source_encoding_name) == 0 &&
2196 strcmp(ec->last_error.destination_encoding,
2197 ec->destination_encoding_name) == 0) {
2198 mesg = rb_sprintf(
"%s from %s to %s",
2200 ec->last_error.source_encoding,
2201 ec->last_error.destination_encoding);
2205 mesg = rb_sprintf(
"%s to %s in conversion from %s",
2207 ec->last_error.destination_encoding,
2208 ec->source_encoding_name);
2209 for (i = 0; i < ec->num_trans; i++) {
2211 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
2212 rb_str_catf(mesg,
" to %s",
2213 ec->elems[i].tc->transcoder->dst_encoding);
2216 exc =
rb_exc_new3(rb_eUndefinedConversionError, mesg);
2217 idx = rb_enc_find_index(ec->last_error.source_encoding);
2219 rb_enc_associate_index(bytes, idx);
2228 int idx = rb_enc_find_index(ec->last_error.source_encoding);
2230 rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2231 idx = rb_enc_find_index(ec->last_error.destination_encoding);
2233 rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2240 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2242 unsigned char **out_start_ptr,
2243 unsigned char **out_pos,
2244 unsigned char **out_stop_ptr)
2246 size_t len = (*out_pos - *out_start_ptr);
2248 rb_raise(rb_eArgError,
"string size too big");
2249 size_t new_len = (
len + max_output) * 2;
2250 *out_start_ptr = resize_destination(destination,
len, new_len);
2251 *out_pos = *out_start_ptr +
len;
2252 *out_stop_ptr = *out_start_ptr + new_len;
2260 const unsigned char *replacement;
2261 const char *repl_enc;
2262 const char *ins_enc;
2265 if (ec->replacement_str)
2272 tr = tc->transcoder;
2273 rb_enc_find(
tr->dst_encoding);
2274 replacement = (
const unsigned char *)get_replacement_character(ins_enc, &
len, &repl_enc);
2277 replacement = (
unsigned char *)
"?";
2282 ec->replacement_str = replacement;
2283 ec->replacement_len =
len;
2284 ec->replacement_bufsize =
len;
2285 ec->replacement_enc = repl_enc;
2286 ec->replacement_allocated = 0;
2292 const unsigned char *str,
size_t len,
const char *encname)
2294 unsigned char *str2;
2295 size_t len2, buf_size2;
2296 const char *encname2;
2300 if (!*encname2 || encoding_equal(encname, encname2)) {
2303 buf_size2 = len2 =
len;
2307 str2 = allocate_converted_string(encname, encname2, str,
len, NULL, 0, &len2, &buf_size2);
2312 if (ec->replacement_allocated) {
2313 SIZED_FREE_N((
char *)ec->replacement_str, ec->replacement_bufsize);
2315 ec->replacement_allocated = 1;
2316 ec->replacement_str = str2;
2317 ec->replacement_len = len2;
2318 ec->replacement_bufsize = buf_size2;
2319 ec->replacement_enc = encname2;
2328 if (make_replacement(ec) == -1)
2339#define hash_fallback rb_hash_aref
2366transcode_loop_fallback_try(
VALUE a)
2370 VALUE ret = args->fallback_func(args->fallback, args->rep);
2372 if (!NIL_OR_UNDEF_P(ret)) {
2380transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2381 const unsigned char *in_stop,
unsigned char *out_stop,
2383 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2384 const char *src_encoding,
2385 const char *dst_encoding,
2393 unsigned char *out_start = *out_pos;
2398 const unsigned char *source_start = *in_pos;
2399 rb_len_t source_len = in_stop - *in_pos;
2406 fallback = rb_hash_aref(ecopts, sym_fallback);
2408 fallback_func = hash_fallback;
2411 fallback_func = proc_fallback;
2414 fallback_func = method_fallback;
2417 fallback_func = aref_fallback;
2420 last_tc = ec->last_tc;
2421 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2427 VALUE rep = rb_enc_str_new(
2428 (
const char *)ec->last_error.error_bytes_start,
2429 ec->last_error.error_bytes_len,
2430 rb_enc_find(ec->last_error.source_encoding));
2434 .fallback_func = fallback_func,
2435 .fallback = fallback,
2440 rep = rb_protect(transcode_loop_fallback_try, (
VALUE)&args, &state);
2449 if ((
const unsigned char *)RSTRING_PTR(source) != source_start ||
2450 RSTRING_LEN(source) != source_len) {
2455 if (!NIL_OR_UNDEF_P(rep)) {
2457 RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
2459 if ((
int)ret == -1) {
2461 rb_raise(rb_eArgError,
"too big fallback string");
2470 exc = make_econv_exception(ec);
2476 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2486transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2487 const unsigned char *in_stop,
unsigned char *out_stop,
2489 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2490 const char *src_encoding,
2491 const char *dst_encoding,
2499 unsigned char *out_start = *out_pos;
2500 const unsigned char *ptr;
2508 last_tc = ec->last_tc;
2509 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2514 unsigned char input_byte;
2515 const unsigned char *p = &input_byte;
2518 if (ptr < in_stop) {
2529 if (&input_byte != p)
2530 ptr += p - &input_byte;
2535 exc = make_econv_exception(ec);
2541 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2562static unsigned char *
2563str_transcoding_resize(
VALUE destination,
size_t len,
size_t new_len)
2565 rb_str_resize(destination, new_len);
2566 return (
unsigned char *)RSTRING_PTR(destination);
2570econv_opts(
VALUE opt,
int ecflags)
2573 int newlineflag = 0;
2575 v = rb_hash_aref(opt, sym_invalid);
2578 else if (v==sym_replace) {
2582 rb_raise(rb_eArgError,
"unknown value for invalid character option");
2585 v = rb_hash_aref(opt, sym_undef);
2588 else if (v==sym_replace) {
2592 rb_raise(rb_eArgError,
"unknown value for undefined character option");
2595 v = rb_hash_aref(opt, sym_replace);
2600 v = rb_hash_aref(opt, sym_xml);
2605 else if (v==sym_attr) {
2609 rb_raise(rb_eArgError,
"unexpected value for xml option: %"PRIsVALUE,
rb_sym2str(v));
2612 rb_raise(rb_eArgError,
"unexpected value for xml option");
2616#ifdef ENABLE_ECONV_NEWLINE_OPTION
2617 v = rb_hash_aref(opt, sym_newline);
2620 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2621 if (v == sym_universal) {
2624 else if (v == sym_crlf) {
2627 else if (v == sym_cr) {
2630 else if (v == sym_lf) {
2634 rb_raise(rb_eArgError,
"unexpected value for newline option: %"PRIsVALUE,
2638 rb_raise(rb_eArgError,
"unexpected value for newline option");
2645 v = rb_hash_aref(opt, sym_universal_newline);
2648 newlineflag |= !
NIL_P(v);
2650 v = rb_hash_aref(opt, sym_crlf_newline);
2653 newlineflag |= !
NIL_P(v);
2655 v = rb_hash_aref(opt, sym_cr_newline);
2658 newlineflag |= !
NIL_P(v);
2660 v = rb_hash_aref(opt, sym_lf_newline);
2663 newlineflag |= !
NIL_P(v);
2665 switch (newlineflag) {
2667 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2668 ecflags |= setflags;
2672 rb_warning(
":newline option precedes other newline options");
2686 if (
NIL_P(opthash)) {
2690 ecflags = econv_opts(opthash, ecflags);
2692 v = rb_hash_aref(opthash, sym_replace);
2695 if (is_broken_string(v)) {
2697 rb_raise(rb_eArgError,
"replacement string is broken: %s as %s",
2699 rb_enc_name(rb_enc_get(v)));
2702 newhash = rb_hash_new();
2703 rb_hash_aset(newhash, sym_replace, v);
2706 v = rb_hash_aref(opthash, sym_fallback);
2708 VALUE h = rb_check_hash_type(v);
2713 newhash = rb_hash_new();
2714 rb_hash_aset(newhash, sym_fallback, v);
2718 if (!
NIL_P(newhash))
2719 rb_hash_freeze(newhash);
2737 if (
NIL_P(opthash)) {
2742 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2743 replacement = rb_hash_aref(opthash, sym_replace);
2746 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2748 if (!
NIL_P(replacement)) {
2753 (
const unsigned char *)RSTRING_PTR(replacement),
2754 RSTRING_LEN(replacement),
2773 if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
2774 !(enc = rb_enc_from_index(encidx))) {
2780 n = rb_enc_name(enc);
2795 const char *sname, *dname;
2796 int sencidx, dencidx;
2798 dencidx = enc_arg(arg1, &dname, &denc);
2801 sencidx = rb_enc_get_index(str);
2802 senc = rb_enc_from_index(sencidx);
2803 sname = rb_enc_name(senc);
2806 sencidx = enc_arg(arg2, &sname, &senc);
2817str_transcode0(
int argc,
VALUE *argv,
VALUE *self,
int ecflags,
VALUE ecopts)
2823 unsigned char *buf, *bp, *sp;
2824 const unsigned char *fromp;
2826 const char *sname, *dname;
2828 int explicitly_invalid_replace = TRUE;
2833 arg1 = rb_enc_default_internal();
2835 if (!ecflags)
return -1;
2836 arg1 = rb_obj_encoding(str);
2839 explicitly_invalid_replace = FALSE;
2846 arg2 = argc<=1 ?
Qnil : argv[1];
2847 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2853 if (senc && senc == denc) {
2856 if (!
NIL_P(ecopts)) {
2857 rep = rb_hash_aref(ecopts, sym_replace);
2859 dest = rb_enc_str_scrub(senc, str, rep);
2860 if (
NIL_P(dest)) dest = str;
2864 return NIL_P(arg2) ? -1 : dencidx;
2866 if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
2867 if (is_ascii_string(str)) {
2871 if (encoding_equal(sname, dname)) {
2872 return NIL_P(arg2) ? -1 : dencidx;
2876 if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) {
2882 if (encoding_equal(sname, dname)) {
2888 fromp = sp = (
unsigned char *)RSTRING_PTR(str);
2889 slen = RSTRING_LEN(str);
2892 bp = (
unsigned char *)RSTRING_PTR(dest);
2894 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts, str);
2895 if (fromp != sp+slen) {
2896 rb_raise(rb_eArgError,
"not fully converted, %"PRIdPTRDIFF
" bytes left", sp+slen-fromp);
2898 buf = (
unsigned char *)RSTRING_PTR(dest);
2904 dencidx = rb_define_dummy_encoding(dname);
2914str_transcode(
int argc,
VALUE *argv,
VALUE *self)
2920 argc =
rb_scan_args(argc, argv,
"02:", NULL, NULL, &opt);
2924 return str_transcode0(argc, argv, self, ecflags, ecopts);
2928str_encode_associate(
VALUE str,
int encidx)
2932 rb_enc_associate_index(str, encidx);
2935 if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
2956str_encode_bang(
int argc,
VALUE *argv,
VALUE str)
2961 rb_check_frozen(str);
2964 encidx = str_transcode(argc, argv, &newstr);
2966 if (encidx < 0)
return str;
2967 if (newstr == str) {
2968 rb_enc_associate_index(str, encidx);
2972 return str_encode_associate(str, encidx);
3039 int encidx = str_transcode(argc, argv, &newstr);
3040 return encoded_dup(newstr, str, encidx);
3049 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
3050 return encoded_dup(newstr, str, encidx);
3054encoded_dup(
VALUE newstr,
VALUE str,
int encidx)
3057 if (newstr == str) {
3059 rb_enc_associate_index(newstr, encidx);
3065 return str_encode_associate(newstr, encidx);
3074econv_free(
void *ptr)
3081econv_memsize(
const void *ptr)
3083 return ptr ? rb_econv_memsize((
rb_econv_t *)ptr) : 0;
3088 {0, econv_free, econv_memsize,},
3089 0, 0, RUBY_TYPED_THREAD_SAFE_FREE
3093econv_s_allocate(
VALUE klass)
3099make_dummy_encoding(
const char *name)
3103 idx = rb_define_dummy_encoding(name);
3104 enc = rb_enc_from_index(idx);
3109make_encoding(
const char *name)
3112 enc = rb_enc_find(name);
3115 if (rb_enc_registered(name)) {
3119 enc = make_dummy_encoding(name);
3127make_encobj(
const char *name)
3129 return rb_enc_from_encoding(make_encoding(name));
3151econv_s_asciicompat_encoding(
VALUE klass,
VALUE arg)
3153 const char *arg_name, *result_name;
3157 enc_arg(&arg, &arg_name, &arg_enc);
3160 result_enc = make_encoding(result_name);
3161 enc = rb_enc_from_encoding(result_enc);
3167econv_args(
int argc,
VALUE *argv,
3169 const char **sname_p,
const char **dname_p,
3174 VALUE opt, flags_v, ecopts;
3176 const char *sname, *dname;
3180 argc =
rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
3182 if (!
NIL_P(flags_v)) {
3184 rb_error_arity(argc + 1, 2, 3);
3189 else if (!
NIL_P(opt)) {
3198 sidx = rb_to_encoding_index(*snamev_p);
3200 senc = rb_enc_from_index(sidx);
3207 didx = rb_to_encoding_index(*dnamev_p);
3209 denc = rb_enc_from_index(didx);
3222 *ecflags_p = ecflags;
3227decorate_convpath(
VALUE convpath,
int ecflags)
3230 const char *decorators[MAX_ECFLAGS_DECORATORS];
3234 num_decorators = decorator_names(ecflags, decorators);
3235 if (num_decorators == -1)
3242 const char *sname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 0)));
3243 const char *dname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 1)));
3246 entry = get_transcoder_entry(sname, dname);
3247 tr = load_transcoder_entry(entry);
3250 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
3251 tr->asciicompat_type == asciicompat_encoder) {
3261 for (i = 0; i < num_decorators; i++)
3268search_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3273 if (
NIL_P(*ary_p)) {
3277 if (DECORATOR_P(sname, dname)) {
3281 v =
rb_assoc_new(make_encobj(sname), make_encobj(dname));
3312econv_s_search_convpath(
int argc,
VALUE *argv,
VALUE klass)
3314 VALUE snamev, dnamev;
3315 const char *sname, *dname;
3321 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3324 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3326 if (
NIL_P(convpath)) {
3333 if (decorate_convpath(convpath, ecflags) == -1) {
3352 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3354 return RTEST(convpath);
3364rb_econv_init_by_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3372 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3379rb_econv_init_by_convpath(
VALUE self,
VALUE convpath,
3380 const char **sname_p,
const char **dname_p,
3388 const char *sname, *dname;
3394 VALUE snamev, dnamev;
3399 rb_raise(rb_eArgError,
"not a 2-element array in convpath");
3401 enc_arg(&snamev, &sname, &senc);
3403 enc_arg(&dnamev, &dname, &denc);
3409 if (DECORATOR_P(sname, dname)) {
3410 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3412 VALUE msg = rb_sprintf(
"decoration failed: %s", dname);
3419 int j = ec->num_trans;
3422 arg.index = ec->num_trans;
3424 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3425 if (ret == -1 || arg.ret == -1) {
3426 VALUE msg = rb_sprintf(
"adding conversion failed: %s to %s", sname, dname);
3434 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3437 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3448 ec->source_encoding_name = *sname_p;
3449 ec->destination_encoding_name = *dname_p;
3569 VALUE snamev, dnamev;
3570 const char *sname, *dname;
3581 ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
3586 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3597 if (!DECORATOR_P(sname, dname)) {
3599 senc = make_dummy_encoding(sname);
3601 denc = make_dummy_encoding(dname);
3606 ec->source_encoding = senc;
3607 ec->destination_encoding = denc;
3625econv_inspect(
VALUE self)
3632 return rb_sprintf(
"#<%s: uninitialized>", cname);
3634 const char *sname = ec->source_encoding_name;
3635 const char *dname = ec->destination_encoding_name;
3637 str = rb_sprintf(
"#<%s: ", cname);
3638 econv_description(sname, dname, ec->flags, str);
3645check_econv(
VALUE self)
3651 rb_raise(
rb_eTypeError,
"uninitialized encoding converter");
3661 return rb_enc_from_encoding(encoding);
3671econv_source_encoding(
VALUE self)
3674 return econv_get_encoding(ec->source_encoding);
3684econv_destination_encoding(
VALUE self)
3687 return econv_get_encoding(ec->destination_encoding);
3713econv_convpath(
VALUE self)
3720 for (i = 0; i < ec->num_trans; i++) {
3723 if (DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
3726 v =
rb_assoc_new(make_encobj(
tr->src_encoding), make_encobj(
tr->dst_encoding));
3743 if (!rb_typeddata_is_kind_of(other, &econv_data_type)) {
3748 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3749 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3751 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3752 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3754 if (ec1->flags != ec2->flags)
return Qfalse;
3755 if (ec1->replacement_enc != ec2->replacement_enc &&
3756 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3758 if (ec1->replacement_len != ec2->replacement_len)
return Qfalse;
3759 if (ec1->replacement_str != ec2->replacement_str &&
3760 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3763 if (ec1->num_trans != ec2->num_trans)
return Qfalse;
3764 for (i = 0; i < ec1->num_trans; i++) {
3765 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3881econv_primitive_convert(
int argc,
VALUE *argv,
VALUE self)
3883 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3886 const unsigned char *ip, *is;
3887 unsigned char *op, *os;
3888 rb_len_t output_byteoffset, output_bytesize;
3892 argc =
rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3894 if (
NIL_P(output_byteoffset_v))
3895 output_byteoffset = 0;
3897 output_byteoffset =
NUM2LEN(output_byteoffset_v);
3899 if (
NIL_P(output_bytesize_v))
3900 output_bytesize = 0;
3902 output_bytesize =
NUM2LEN(output_bytesize_v);
3904 if (!
NIL_P(flags_v)) {
3906 rb_error_arity(argc + 1, 2, 5);
3910 else if (!
NIL_P(opt)) {
3913 v = rb_hash_aref(opt, sym_partial_input);
3916 v = rb_hash_aref(opt, sym_after_output);
3927 rb_str_modify(output);
3929 if (
NIL_P(output_bytesize_v)) {
3932 if (!
NIL_P(input) && output_bytesize < RSTRING_LEN(input))
3933 output_bytesize = RSTRING_LEN(input);
3938 if (
NIL_P(output_byteoffset_v))
3939 output_byteoffset = RSTRING_LEN(output);
3941 if (output_byteoffset < 0)
3942 rb_raise(rb_eArgError,
"negative output_byteoffset");
3944 if (RSTRING_LEN(output) < output_byteoffset)
3945 rb_raise(rb_eArgError,
"output_byteoffset too big");
3947 if (output_bytesize < 0)
3948 rb_raise(rb_eArgError,
"negative output_bytesize");
3950 output_byteend = (
rb_ulen_t)output_byteoffset +
3953 if (output_byteend < (
rb_ulen_t)output_byteoffset ||
3955 rb_raise(rb_eArgError,
"output_byteoffset+output_bytesize too big");
3958 rb_str_resize(output, output_byteend);
3964 ip = (
const unsigned char *)RSTRING_PTR(input);
3965 is = ip + RSTRING_LEN(input);
3968 op = (
unsigned char *)RSTRING_PTR(output) + output_byteoffset;
3969 os = op + output_bytesize;
3973 if (!
NIL_P(input)) {
3979 rb_raise(rb_eArgError,
"too long conversion result");
3980 output_bytesize *= 2;
3981 output_byteoffset_v =
Qnil;
3985 if (ec->destination_encoding) {
3986 rb_enc_associate(output, ec->destination_encoding);
3989 return econv_result_to_symbol(res);
4027econv_convert(
VALUE self,
VALUE source_string)
4045 ret = econv_primitive_convert(ac, av, self);
4047 if (ret == sym_invalid_byte_sequence ||
4048 ret == sym_undefined_conversion ||
4049 ret == sym_incomplete_input) {
4050 VALUE exc = make_econv_exception(ec);
4054 if (ret == sym_finished) {
4055 rb_raise(rb_eArgError,
"converter already finished");
4058 if (ret != sym_source_buffer_empty) {
4059 rb_bug(
"unexpected result of econv_primitive_convert");
4077econv_finish(
VALUE self)
4093 ret = econv_primitive_convert(ac, av, self);
4095 if (ret == sym_invalid_byte_sequence ||
4096 ret == sym_undefined_conversion ||
4097 ret == sym_incomplete_input) {
4098 VALUE exc = make_econv_exception(ec);
4102 if (ret != sym_finished) {
4103 rb_bug(
"unexpected result of econv_primitive_convert");
4185econv_primitive_errinfo(
VALUE self)
4193 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4196 if (ec->last_error.source_encoding)
4199 if (ec->last_error.destination_encoding)
4202 if (ec->last_error.error_bytes_start) {
4203 rb_ary_store(ary, 3,
rb_str_new((
const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4204 rb_ary_store(ary, 4,
rb_str_new((
const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4245 const char *insert_enc;
4253 string =
rb_str_encode(
string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0,
Qnil);
4255 ret =
rb_econv_insert_output(ec, (
const unsigned char *)RSTRING_PTR(
string), RSTRING_LEN(
string), insert_enc);
4257 rb_raise(rb_eArgError,
"too big string");
4288econv_putback(
int argc,
VALUE *argv,
VALUE self)
4301 if (putbackable < n)
4308 if (ec->source_encoding) {
4309 rb_enc_associate(str, ec->source_encoding);
4336econv_last_error(
VALUE self)
4341 exc = make_econv_exception(ec);
4360econv_get_replacement(
VALUE self)
4366 ret = make_replacement(ec);
4368 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4371 enc = rb_enc_find(ec->replacement_enc);
4372 return rb_enc_str_new((
const char *)ec->replacement_str, (
rb_len_t)ec->replacement_len, enc);
4394 enc = rb_enc_get(
string);
4397 (
const unsigned char *)RSTRING_PTR(
string),
4398 RSTRING_LEN(
string),
4403 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4412 return make_econv_exception(ec);
4420 exc = make_econv_exception(ec);
4433ecerr_source_encoding_name(
VALUE self)
4435 return rb_attr_get(self, id_source_encoding_name);
4459ecerr_source_encoding(
VALUE self)
4461 return rb_attr_get(self, id_source_encoding);
4471ecerr_destination_encoding_name(
VALUE self)
4473 return rb_attr_get(self, id_destination_encoding_name);
4483ecerr_destination_encoding(
VALUE self)
4485 return rb_attr_get(self, id_destination_encoding);
4504ecerr_error_char(
VALUE self)
4506 return rb_attr_get(self, id_error_char);
4525ecerr_error_bytes(
VALUE self)
4527 return rb_attr_get(self, id_error_bytes);
4537ecerr_readagain_bytes(
VALUE self)
4539 return rb_attr_get(self, id_readagain_bytes);
4567ecerr_incomplete_input(
VALUE self)
4569 return rb_attr_get(self, id_incomplete_input);
4597 transcoder_table = st_init_strcasetable();
4600 id_destination_encoding_name =
rb_intern_const(
"destination_encoding_name");
4626 sym_lf_newline =
ID2SYM(rb_intern(
"lf_newline"));
4629#ifdef ENABLE_ECONV_NEWLINE_OPTION
4641InitVM_transcode(
void)
4657 rb_define_method(rb_cEncodingConverter,
"source_encoding", econv_source_encoding, 0);
4658 rb_define_method(rb_cEncodingConverter,
"destination_encoding", econv_destination_encoding, 0);
4659 rb_define_method(rb_cEncodingConverter,
"primitive_convert", econv_primitive_convert, -1);
4662 rb_define_method(rb_cEncodingConverter,
"primitive_errinfo", econv_primitive_errinfo, 0);
4663 rb_define_method(rb_cEncodingConverter,
"insert_output", econv_insert_output, 1);
4665 rb_define_method(rb_cEncodingConverter,
"last_error", econv_last_error, 0);
4666 rb_define_method(rb_cEncodingConverter,
"replacement", econv_get_replacement, 0);
4667 rb_define_method(rb_cEncodingConverter,
"replacement=", econv_set_replacement, 1);
4745 rb_define_method(rb_eUndefinedConversionError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4746 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4747 rb_define_method(rb_eUndefinedConversionError,
"source_encoding", ecerr_source_encoding, 0);
4748 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding", ecerr_destination_encoding, 0);
4749 rb_define_method(rb_eUndefinedConversionError,
"error_char", ecerr_error_char, 0);
4751 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4752 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4753 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding", ecerr_source_encoding, 0);
4754 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding", ecerr_destination_encoding, 0);
4755 rb_define_method(rb_eInvalidByteSequenceError,
"error_bytes", ecerr_error_bytes, 0);
4756 rb_define_method(rb_eInvalidByteSequenceError,
"readagain_bytes", ecerr_readagain_bytes, 0);
4757 rb_define_method(rb_eInvalidByteSequenceError,
"incomplete_input?", ecerr_incomplete_input, 0);
ruby_coderange_type
What rb_enc_str_coderange() returns.
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
#define rb_str_new2
Old name of rb_str_new_cstr.
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
#define ALLOC
Old name of RB_ALLOC.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
#define rb_str_cat2
Old name of rb_str_cat_cstr.
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
#define rb_ary_new4
Old name of rb_ary_new_from_values.
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
#define xmalloc
Old name of ruby_xmalloc.
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
#define rb_exc_new3
Old name of rb_exc_new_str.
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
#define NUM2INT
Old name of RB_NUM2INT.
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
#define ECONV_LF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_LF_NEWLINE_DECORATOR.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
#define NUM2LEN
Old name of RB_NUM2LEN.
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
#define rb_ary_new2
Old name of rb_ary_new_capa.
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
VALUE rb_eTypeError
TypeError exception.
VALUE rb_eRuntimeError
RuntimeError exception.
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
VALUE rb_eEncodingError
EncodingError exception.
void rb_warning(const char *fmt,...)
Issues a warning.
VALUE rb_cObject
Object class.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cEncoding
Encoding class.
VALUE rb_cString
String class.
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
rb_len_t rb_str_coderange_scan_restartable(const char *str, const char *end, rb_encoding *enc, int *cr)
Scans the passed string until it finds something odd.
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, rb_len_t byteoff, rb_len_t bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, rb_len_t byteoff, rb_len_t bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
rb_econv_result_t
return value of rb_econv_convert()
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
@ econv_finished
The conversion stopped after converting everything.
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
@ econv_source_buffer_empty
The conversion stopped because there is no input.
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, rb_len_t bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
VALUE rb_ary_entry(VALUE ary, rb_len_t off)
Queries an element of an array.
void rb_ary_store(VALUE ary, rb_len_t key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_assoc_new(VALUE car, VALUE cdr)
Identical to rb_ary_new_from_values(), except it expects exactly two parameters.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
void rb_str_modify_expand(VALUE str, rb_len_t capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
VALUE rb_str_tmp_new(rb_len_t len)
Allocates a "temporary" string.
#define rb_str_new(str, len)
Allocates an instance of rb_cString.
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
void rb_str_set_len(VALUE str, rb_len_t len)
Overwrites the length of the string.
VALUE rb_str_dup(VALUE str)
Duplicates a string.
VALUE rb_str_drop_bytes(VALUE str, rb_len_t len)
Shrinks the given string for the given number of bytes.
VALUE rb_str_buf_new(rb_len_t capa)
Allocates a "string buffer".
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
int off
Offset inside of ptr.
int len
Length of the buffer.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define ALLOCA_N(type, n)
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
#define RARRAY_LEN
Just another name of rb_array_len.
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
#define RARRAY_AREF(a, i)
References an object in an array.
#define StringValue(v)
Ensures that the parameter object is a String.
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
#define DATA_PTR(obj)
Convenient casting macro for backward compatibility.
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
#define InitVM(ext)
This macro is for internal use.
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
#define RB_LEN_MAX
Maximum possible value that a rb_len_t can take.
unsigned long rb_ulen_t
An unsigned integer type that has the same width with rb_len_t.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.