12 #include "ruby/internal/config.h"
17 #include "internal/array.h"
18 #include "internal/inits.h"
19 #include "internal/object.h"
20 #include "internal/string.h"
21 #include "internal/transcode.h"
24 #include "transcode_data.h"
27 #define ENABLE_ECONV_NEWLINE_OPTION 1
30 static VALUE rb_eUndefinedConversionError;
31 static VALUE rb_eInvalidByteSequenceError;
32 static VALUE rb_eConverterNotFoundError;
34 VALUE rb_cEncodingConverter;
36 static ID id_destination_encoding;
37 static ID id_destination_encoding_name;
38 static ID id_error_bytes;
39 static ID id_error_char;
40 static ID id_incomplete_input;
41 static ID id_readagain_bytes;
42 static ID id_source_encoding;
43 static ID id_source_encoding_name;
45 static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
46 static VALUE sym_xml, sym_text, sym_attr;
47 static VALUE sym_universal_newline;
48 static VALUE sym_crlf_newline;
49 static VALUE sym_cr_newline;
50 static VALUE sym_lf_newline;
51 #ifdef ENABLE_ECONV_NEWLINE_OPTION
52 static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
54 static VALUE sym_partial_input;
56 static VALUE sym_invalid_byte_sequence;
57 static VALUE sym_undefined_conversion;
58 static VALUE sym_destination_buffer_full;
59 static VALUE sym_source_buffer_empty;
60 static VALUE sym_finished;
61 static VALUE sym_after_output;
62 static VALUE sym_incomplete_input;
64 static unsigned char *
65 allocate_converted_string(
const char *sname,
const char *dname,
66 const unsigned char *str,
size_t len,
67 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
78 unsigned int next_table;
80 unsigned char next_byte;
81 unsigned int output_index;
83 ssize_t recognized_len;
84 ssize_t readagain_len;
99 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
100 double dummy_for_alignment;
103 #define TRANSCODING_READBUF(tc) \
104 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
105 (tc)->readbuf.ary : \
107 #define TRANSCODING_WRITEBUF(tc) \
108 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
109 (tc)->writebuf.ary : \
111 #define TRANSCODING_WRITEBUF_SIZE(tc) \
112 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
113 sizeof((tc)->writebuf.ary) : \
114 (size_t)(tc)->transcoder->max_output)
115 #define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
116 #define TRANSCODING_STATE(tc) \
117 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
123 unsigned char *out_buf_start;
124 unsigned char *out_data_start;
125 unsigned char *out_data_end;
126 unsigned char *out_buf_end;
134 const char *source_encoding_name;
135 const char *destination_encoding_name;
137 const unsigned char *replacement_str;
138 size_t replacement_len;
139 const char *replacement_enc;
141 unsigned char *in_buf_start;
142 unsigned char *in_data_start;
143 unsigned char *in_data_end;
144 unsigned char *in_buf_end;
146 int replacement_allocated;
156 const char *source_encoding;
157 const char *destination_encoding;
158 const unsigned char *error_bytes_start;
159 size_t error_bytes_len;
160 size_t readagain_len;
173 #define DECORATOR_P(sname, dname) (*(sname) == '\0')
185 free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
192 free_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
194 st_foreach((
void *)val, free_inner_transcode_i, 0);
195 st_free_table((
void *)val);
200 rb_free_transcoder_table(
void)
202 st_foreach(transcoder_table, free_transcode_i, 0);
203 st_free_table(transcoder_table);
207 make_transcoder_entry(
const char *sname,
const char *dname)
212 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
213 val = (st_data_t)st_init_strcasetable();
214 st_add_direct(transcoder_table, (st_data_t)sname, val);
217 if (!st_lookup(table2, (st_data_t)dname, &val)) {
219 entry->sname = sname;
220 entry->dname = dname;
222 entry->transcoder = NULL;
223 val = (st_data_t)entry;
224 st_add_direct(table2, (st_data_t)dname, val);
230 get_transcoder_entry(
const char *sname,
const char *dname)
235 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
239 if (!st_lookup(table2, (st_data_t)dname, &val)) {
248 const char *
const sname =
tr->src_encoding;
249 const char *
const dname =
tr->dst_encoding;
253 entry = make_transcoder_entry(sname, dname);
254 if (entry->transcoder) {
259 entry->transcoder =
tr;
263 declare_transcoder(
const char *sname,
const char *dname,
const char *lib)
267 entry = make_transcoder_entry(sname, dname);
271 static const char transcoder_lib_prefix[] =
"enc/trans/";
274 rb_declare_transcoder(
const char *enc1,
const char *enc2,
const char *lib)
279 declare_transcoder(enc1, enc2, lib);
282 #define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
293 const char *base_enc;
297 transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
299 const char *dname = (
const char *)key;
303 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
310 *bfs->queue_last_ptr = q;
311 bfs->queue_last_ptr = &q->next;
313 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
318 transcode_search_path(
const char *sname,
const char *dname,
319 void (*callback)(
const char *sname,
const char *dname,
int depth,
void *arg),
329 if (encoding_equal(sname, dname))
335 bfs.queue_last_ptr = &q->next;
338 bfs.visited = st_init_strcasetable();
339 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
345 bfs.queue_last_ptr = &bfs.queue;
347 if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) {
353 if (st_lookup(table2, (st_data_t)dname, &val)) {
354 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
360 bfs.base_enc = q->enc;
361 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
376 const char *enc = dname;
380 st_lookup(bfs.visited, (st_data_t)enc, &val);
384 enc = (
const char *)val;
389 st_lookup(bfs.visited, (st_data_t)enc, &val);
392 callback((
const char *)val, enc, --depth, arg);
393 enc = (
const char *)val;
397 st_free_table(bfs.visited);
402 int rb_require_internal_silent(
VALUE fname);
407 if (entry->transcoder)
408 return entry->transcoder;
411 const char *
const lib = entry->lib;
412 const size_t len = strlen(lib);
413 const size_t total_len =
sizeof(transcoder_lib_prefix) - 1 +
len;
417 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
418 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib,
len);
421 rb_require_internal_silent(fn);
424 if (entry->transcoder)
425 return entry->transcoder;
431 get_replacement_character(
const char *encname,
size_t *len_ret,
const char **repl_encname_ptr)
433 if (encoding_equal(encname,
"UTF-8")) {
435 *repl_encname_ptr =
"UTF-8";
436 return "\xEF\xBF\xBD";
440 *repl_encname_ptr =
"US-ASCII";
449 static const unsigned char *
451 const unsigned char *in_start,
452 const unsigned char *inchar_start,
453 const unsigned char *in_p,
454 size_t *char_len_ptr)
456 const unsigned char *
ptr;
457 if (inchar_start - in_start < tc->recognized_len) {
458 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
459 inchar_start,
unsigned char, in_p - inchar_start);
460 ptr = TRANSCODING_READBUF(tc);
463 ptr = inchar_start - tc->recognized_len;
465 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
470 transcode_restartable0(
const unsigned char **in_pos,
unsigned char **out_pos,
471 const unsigned char *in_stop,
unsigned char *out_stop,
476 int unitlen =
tr->input_unit_length;
477 ssize_t readagain_len = 0;
479 const unsigned char *inchar_start;
480 const unsigned char *in_p;
482 unsigned char *out_p;
484 in_p = inchar_start = *in_pos;
488 #define SUSPEND(ret, num) \
490 tc->resume_position = (num); \
491 if (0 < in_p - inchar_start) \
492 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
493 inchar_start, unsigned char, in_p - inchar_start); \
496 tc->recognized_len += in_p - inchar_start; \
497 if (readagain_len) { \
498 tc->recognized_len -= readagain_len; \
499 tc->readagain_len = readagain_len; \
502 resume_label ## num:; \
504 #define SUSPEND_OBUF(num) \
506 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
509 #define SUSPEND_AFTER_OUTPUT(num) \
510 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
511 SUSPEND(econv_after_output, num); \
514 #define next_table (tc->next_table)
515 #define next_info (tc->next_info)
516 #define next_byte (tc->next_byte)
517 #define writebuf_len (tc->writebuf_len)
518 #define writebuf_off (tc->writebuf_off)
520 switch (tc->resume_position) {
522 case 1:
goto resume_label1;
523 case 2:
goto resume_label2;
524 case 3:
goto resume_label3;
525 case 4:
goto resume_label4;
526 case 5:
goto resume_label5;
527 case 6:
goto resume_label6;
528 case 7:
goto resume_label7;
529 case 8:
goto resume_label8;
530 case 9:
goto resume_label9;
531 case 10:
goto resume_label10;
532 case 11:
goto resume_label11;
533 case 12:
goto resume_label12;
534 case 13:
goto resume_label13;
535 case 14:
goto resume_label14;
536 case 15:
goto resume_label15;
537 case 16:
goto resume_label16;
538 case 17:
goto resume_label17;
539 case 18:
goto resume_label18;
540 case 19:
goto resume_label19;
541 case 20:
goto resume_label20;
542 case 21:
goto resume_label21;
543 case 22:
goto resume_label22;
544 case 23:
goto resume_label23;
545 case 24:
goto resume_label24;
546 case 25:
goto resume_label25;
547 case 26:
goto resume_label26;
548 case 27:
goto resume_label27;
549 case 28:
goto resume_label28;
550 case 29:
goto resume_label29;
551 case 30:
goto resume_label30;
552 case 31:
goto resume_label31;
553 case 32:
goto resume_label32;
554 case 33:
goto resume_label33;
555 case 34:
goto resume_label34;
560 tc->recognized_len = 0;
561 next_table =
tr->conv_tree_start;
563 SUSPEND_AFTER_OUTPUT(24);
565 if (in_stop <= in_p) {
572 #define BYTE_ADDR(index) (tr->byte_array + (index))
573 #define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
574 #define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
575 #define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
576 #define BL_MIN_BYTE (BL_BASE[0])
577 #define BL_MAX_BYTE (BL_BASE[1])
578 #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
579 #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
581 next_byte = (
unsigned char)*in_p++;
583 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
586 next_info = (
VALUE)BL_ACTION(next_byte);
589 switch (next_info & 0x1F) {
592 const unsigned char *p = inchar_start;
595 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (
unsigned char)*p++;
597 writebuf_len = writebuf_off;
599 while (writebuf_off < writebuf_len) {
601 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
605 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
606 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
607 SUSPEND_AFTER_OUTPUT(25);
608 while (in_p >= in_stop) {
613 next_byte = (
unsigned char)*in_p++;
614 next_table = (
unsigned int)next_info;
619 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
622 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
623 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
626 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
627 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
628 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
631 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
632 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
633 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
634 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
637 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
638 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
639 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
640 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
643 tc->output_index = 0;
644 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
645 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
650 next_info = (
VALUE)(*
tr->func_ii)(TRANSCODING_STATE(tc), next_info);
654 const unsigned char *char_start;
656 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
657 next_info = (
VALUE)(*
tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
662 if (
tr->max_output <= out_stop - out_p)
663 out_p +=
tr->func_io(TRANSCODING_STATE(tc),
664 next_info, out_p, out_stop - out_p);
666 writebuf_len =
tr->func_io(TRANSCODING_STATE(tc),
668 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
670 while (writebuf_off < writebuf_len) {
672 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
678 const unsigned char *char_start;
681 if (
tr->max_output <= out_stop - out_p) {
682 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
683 out_p +=
tr->func_so(TRANSCODING_STATE(tc),
684 char_start, (
size_t)char_len,
685 out_p, out_stop - out_p);
688 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
689 writebuf_len =
tr->func_so(TRANSCODING_STATE(tc),
690 char_start, (
size_t)char_len,
691 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
693 while (writebuf_off < writebuf_len) {
695 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
702 const unsigned char *char_start;
705 if (
tr->max_output <= out_stop - out_p) {
706 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
707 out_p +=
tr->func_sio(TRANSCODING_STATE(tc),
708 char_start, (
size_t)char_len, next_info,
709 out_p, out_stop - out_p);
712 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
713 writebuf_len =
tr->func_sio(TRANSCODING_STATE(tc),
714 char_start, (
size_t)char_len, next_info,
715 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
717 while (writebuf_off < writebuf_len) {
719 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
725 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
726 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
727 SUSPEND_AFTER_OUTPUT(26);
728 while ((opt &
ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
732 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
736 in_p = inchar_start + (unitlen - tc->recognized_len);
742 invalid_len = tc->recognized_len + (in_p - inchar_start);
743 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
744 readagain_len = invalid_len - discard_len;
768 if (
tr->finish_func) {
770 if (
tr->max_output <= out_stop - out_p) {
771 out_p +=
tr->finish_func(TRANSCODING_STATE(tc),
772 out_p, out_stop - out_p);
775 writebuf_len =
tr->finish_func(TRANSCODING_STATE(tc),
776 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
778 while (writebuf_off < writebuf_len) {
780 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
795 transcode_restartable(
const unsigned char **in_pos,
unsigned char **out_pos,
796 const unsigned char *in_stop,
unsigned char *out_stop,
800 if (tc->readagain_len) {
801 unsigned char *readagain_buf =
ALLOCA_N(
unsigned char, tc->readagain_len);
802 const unsigned char *readagain_pos = readagain_buf;
803 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
806 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
807 unsigned char, tc->readagain_len);
808 tc->readagain_len = 0;
809 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|
ECONV_PARTIAL_INPUT);
811 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
812 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
813 tc->readagain_len += readagain_stop - readagain_pos;
817 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
828 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
830 if (
tr->state_init_func) {
831 (
tr->state_init_func)(TRANSCODING_STATE(tc));
833 tc->resume_position = 0;
834 tc->recognized_len = 0;
835 tc->readagain_len = 0;
836 tc->writebuf_len = 0;
837 tc->writebuf_off = 0;
838 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
839 tc->readbuf.ptr =
xmalloc(
tr->max_input);
841 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
842 tc->writebuf.ptr =
xmalloc(
tr->max_output);
849 const unsigned char **input_ptr,
const unsigned char *input_stop,
850 unsigned char **output_ptr,
unsigned char *output_stop,
853 return transcode_restartable(
854 input_ptr, output_ptr,
855 input_stop, output_stop,
863 if (
tr->state_fini_func) {
864 (
tr->state_fini_func)(TRANSCODING_STATE(tc));
866 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
867 xfree(tc->state.ptr);
868 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input)
869 xfree(tc->readbuf.ptr);
870 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output)
871 xfree(tc->writebuf.ptr);
881 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
882 size +=
tr->state_size;
884 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
885 size +=
tr->max_input;
887 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
888 size +=
tr->max_output;
894 rb_econv_alloc(
int n_hint)
903 ec->source_encoding_name = NULL;
904 ec->destination_encoding_name = NULL;
906 ec->replacement_str = NULL;
907 ec->replacement_len = 0;
908 ec->replacement_enc = NULL;
909 ec->replacement_allocated = 0;
910 ec->in_buf_start = NULL;
911 ec->in_data_start = NULL;
912 ec->in_data_end = NULL;
913 ec->in_buf_end = NULL;
914 ec->num_allocated = n_hint;
917 ec->num_finished = 0;
920 ec->last_error.error_tc = NULL;
921 ec->last_error.source_encoding = NULL;
922 ec->last_error.destination_encoding = NULL;
923 ec->last_error.error_bytes_start = NULL;
924 ec->last_error.error_bytes_len = 0;
925 ec->last_error.readagain_len = 0;
926 ec->source_encoding = NULL;
927 ec->destination_encoding = NULL;
938 if (ec->num_trans == ec->num_allocated) {
939 n = ec->num_allocated * 2;
941 ec->num_allocated = n;
948 ec->elems[i].tc = rb_transcoding_open_by_transcoder(
tr, 0);
949 ec->elems[i].out_buf_start = p;
950 ec->elems[i].out_buf_end = p + bufsize;
951 ec->elems[i].out_data_start = p;
952 ec->elems[i].out_data_end = p;
957 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
958 for (j = ec->num_trans-1; i <= j; j--) {
961 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
976 for (i = 0; i < n; i++) {
978 tr = load_transcoder_entry(entries[i]);
983 ec = rb_econv_alloc(n);
985 for (i = 0; i < n; i++) {
987 ret = rb_econv_add_transcoder_at(ec,
tr, ec->num_trans);
1003 trans_open_i(
const char *sname,
const char *dname,
int depth,
void *arg)
1007 if (!toarg->entries) {
1010 toarg->entries[depth] = get_transcoder_entry(sname, dname);
1014 rb_econv_open0(
const char *sname,
const char *dname,
int ecflags)
1025 if (*sname ==
'\0' && *dname ==
'\0') {
1032 toarg.entries = NULL;
1033 toarg.num_additional = 0;
1034 num_trans = transcode_search_path(sname, dname, trans_open_i, (
void *)&toarg);
1035 entries = toarg.entries;
1036 if (num_trans < 0) {
1042 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1047 ec->flags = ecflags;
1048 ec->source_encoding_name = sname;
1049 ec->destination_encoding_name = dname;
1054 #define MAX_ECFLAGS_DECORATORS 32
1057 decorator_names(
int ecflags,
const char **decorators_ret)
1079 decorators_ret[num_decorators++] =
"xml_text_escape";
1081 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1083 decorators_ret[num_decorators++] =
"xml_attr_quote";
1086 decorators_ret[num_decorators++] =
"crlf_newline";
1088 decorators_ret[num_decorators++] =
"cr_newline";
1090 decorators_ret[num_decorators++] =
"lf_newline";
1092 decorators_ret[num_decorators++] =
"universal_newline";
1094 return num_decorators;
1102 const char *decorators[MAX_ECFLAGS_DECORATORS];
1105 num_decorators = decorator_names(ecflags, decorators);
1106 if (num_decorators == -1)
1113 for (i = 0; i < num_decorators; i++)
1126 const unsigned char **input_ptr,
const unsigned char *input_stop,
1127 unsigned char **output_ptr,
unsigned char *output_stop,
1134 const unsigned char **ipp, *is, *iold;
1135 unsigned char **opp, *os, *oold;
1141 for (i = start; i < ec->num_trans; i++) {
1150 ipp = (
const unsigned char **)&prev_te->out_data_start;
1151 is = prev_te->out_data_end;
1154 if (i == ec->num_trans-1) {
1159 if (te->out_buf_start != te->out_data_start) {
1160 ssize_t
len = te->out_data_end - te->out_data_start;
1161 ssize_t
off = te->out_data_start - te->out_buf_start;
1162 MEMMOVE(te->out_buf_start, te->out_data_start,
unsigned char,
len);
1163 te->out_data_start = te->out_buf_start;
1164 te->out_data_end -=
off;
1166 opp = &te->out_data_end;
1167 os = te->out_buf_end;
1171 if (ec->num_finished != i)
1181 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1182 if (iold != *ipp || oold != *opp)
1197 ec->num_finished = i+1;
1207 const unsigned char **input_ptr,
const unsigned char *input_stop,
1208 unsigned char **output_ptr,
unsigned char *output_stop,
1210 int *result_position_ptr)
1213 int needreport_index;
1216 unsigned char empty_buf;
1217 unsigned char *empty_ptr = &empty_buf;
1220 input_ptr = (
const unsigned char **)&empty_ptr;
1221 input_stop = empty_ptr;
1225 output_ptr = &empty_ptr;
1226 output_stop = empty_ptr;
1232 for (i = ec->num_trans-1; 0 <= i; i--) {
1233 switch (ec->elems[i].last_result) {
1240 goto found_needreport;
1247 rb_bug(
"unexpected transcode last result");
1257 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1259 result_position_ptr);
1271 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1272 sweep_start = needreport_index + 1;
1273 }
while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1275 for (i = ec->num_trans-1; 0 <= i; i--) {
1284 if (result_position_ptr)
1285 *result_position_ptr = i;
1289 if (result_position_ptr)
1290 *result_position_ptr = -1;
1296 const unsigned char **input_ptr,
const unsigned char *input_stop,
1297 unsigned char **output_ptr,
unsigned char *output_stop,
1301 int result_position;
1304 memset(&ec->last_error, 0,
sizeof(ec->last_error));
1306 if (ec->num_trans == 0) {
1308 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1309 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1310 len = output_stop - *output_ptr;
1311 memcpy(*output_ptr, ec->in_data_start,
len);
1312 *output_ptr = output_stop;
1313 ec->in_data_start +=
len;
1317 len = ec->in_data_end - ec->in_data_start;
1318 memcpy(*output_ptr, ec->in_data_start,
len);
1320 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1326 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1327 len = output_stop - *output_ptr;
1330 len = input_stop - *input_ptr;
1333 *(*output_ptr)++ = *(*input_ptr)++;
1337 memcpy(*output_ptr, *input_ptr,
len);
1340 if (*input_ptr != input_stop)
1349 if (ec->elems[ec->num_trans-1].out_data_start) {
1350 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1351 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1352 if (data_start != data_end) {
1354 if (output_stop - *output_ptr < data_end - data_start) {
1355 len = output_stop - *output_ptr;
1356 memcpy(*output_ptr, data_start,
len);
1357 *output_ptr = output_stop;
1358 ec->elems[ec->num_trans-1].out_data_start +=
len;
1362 len = data_end - data_start;
1363 memcpy(*output_ptr, data_start,
len);
1365 ec->elems[ec->num_trans-1].out_data_start =
1366 ec->elems[ec->num_trans-1].out_data_end =
1367 ec->elems[ec->num_trans-1].out_buf_start;
1372 if (ec->in_buf_start &&
1373 ec->in_data_start != ec->in_data_end) {
1374 res = rb_trans_conv(ec, (
const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1382 *input_ptr != input_stop) {
1383 input_stop = *input_ptr;
1384 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1389 ec->num_trans == 1) {
1390 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1395 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1400 ec->last_error.result = res;
1405 ec->last_error.error_tc = error_tc;
1406 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1407 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1408 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1409 ec->last_error.error_bytes_len = error_tc->recognized_len;
1410 ec->last_error.readagain_len = error_tc->readagain_len;
1416 static int output_replacement_character(
rb_econv_t *ec);
1422 unsigned char utfbuf[1024];
1423 const unsigned char *utf;
1425 int utf_allocated = 0;
1426 char charef_buf[16];
1427 const unsigned char *p;
1429 if (encoding_equal(ec->last_error.source_encoding,
"UTF-32BE")) {
1430 utf = ec->last_error.error_bytes_start;
1431 utf_len = ec->last_error.error_bytes_len;
1434 utf = allocate_converted_string(ec->last_error.source_encoding,
"UTF-32BE",
1435 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1436 utfbuf,
sizeof(utfbuf),
1440 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1444 if (utf_len % 4 != 0)
1448 while (4 <= utf_len) {
1454 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1476 const unsigned char **input_ptr,
const unsigned char *input_stop,
1477 unsigned char **output_ptr,
unsigned char *output_stop,
1482 unsigned char empty_buf;
1483 unsigned char *empty_ptr = &empty_buf;
1488 input_ptr = (
const unsigned char **)&empty_ptr;
1489 input_stop = empty_ptr;
1493 output_ptr = &empty_ptr;
1494 output_stop = empty_ptr;
1498 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1506 if (output_replacement_character(ec) == 0)
1517 if (output_replacement_character(ec) == 0)
1522 if (output_hex_charref(ec) == 0)
1540 tr = tc->transcoder;
1542 if (
tr->asciicompat_type == asciicompat_encoder)
1543 return tr->src_encoding;
1544 return tr->dst_encoding;
1547 static unsigned char *
1548 allocate_converted_string(
const char *sname,
const char *dname,
1549 const unsigned char *str,
size_t len,
1550 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1551 size_t *dst_len_ptr)
1553 unsigned char *dst_str;
1560 const unsigned char *sp;
1564 dst_bufsize = caller_dst_bufsize;
1574 dst_str = caller_dst_buf;
1576 dst_str =
xmalloc(dst_bufsize);
1579 dp = dst_str+dst_len;
1581 dst_len = dp - dst_str;
1583 if (SIZE_MAX/2 < dst_bufsize) {
1587 if (dst_str == caller_dst_buf) {
1590 memcpy(tmp, dst_str, dst_bufsize/2);
1594 dst_str =
xrealloc(dst_str, dst_bufsize);
1596 dp = dst_str+dst_len;
1598 dst_len = dp - dst_str;
1604 *dst_len_ptr = dst_len;
1608 if (dst_str != caller_dst_buf)
1617 const unsigned char *str,
size_t len,
const char *str_encoding)
1620 unsigned char insert_buf[4096];
1621 const unsigned char *insert_str = NULL;
1624 int last_trans_index;
1627 unsigned char **buf_start_p;
1628 unsigned char **data_start_p;
1629 unsigned char **data_end_p;
1630 unsigned char **buf_end_p;
1639 if (encoding_equal(insert_encoding, str_encoding)) {
1644 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1645 str,
len, insert_buf,
sizeof(insert_buf), &insert_len);
1646 if (insert_str == NULL)
1652 last_trans_index = ec->num_trans-1;
1653 if (ec->num_trans == 0) {
1655 buf_start_p = &ec->in_buf_start;
1656 data_start_p = &ec->in_data_start;
1657 data_end_p = &ec->in_data_end;
1658 buf_end_p = &ec->in_buf_end;
1660 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1661 tc = ec->elems[last_trans_index].tc;
1662 need += tc->readagain_len;
1663 if (need < insert_len)
1665 if (last_trans_index == 0) {
1666 buf_start_p = &ec->in_buf_start;
1667 data_start_p = &ec->in_data_start;
1668 data_end_p = &ec->in_data_end;
1669 buf_end_p = &ec->in_buf_end;
1673 buf_start_p = &ee->out_buf_start;
1674 data_start_p = &ee->out_data_start;
1675 data_end_p = &ee->out_data_end;
1676 buf_end_p = &ee->out_buf_end;
1681 buf_start_p = &ee->out_buf_start;
1682 data_start_p = &ee->out_data_start;
1683 data_end_p = &ee->out_data_end;
1684 buf_end_p = &ee->out_buf_end;
1685 tc = ec->elems[last_trans_index].tc;
1688 if (*buf_start_p == NULL) {
1689 unsigned char *buf =
xmalloc(need);
1691 *data_start_p = buf;
1693 *buf_end_p = buf+need;
1695 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1696 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1697 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1698 *data_start_p = *buf_start_p;
1699 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1701 size_t s = (*data_end_p - *buf_start_p) + need;
1705 *data_start_p = buf;
1706 *data_end_p = buf + (*data_end_p - *buf_start_p);
1708 *buf_end_p = buf + s;
1712 memcpy(*data_end_p, insert_str, insert_len);
1713 *data_end_p += insert_len;
1714 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1715 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1716 *data_end_p += tc->readagain_len;
1717 tc->readagain_len = 0;
1720 if (insert_str != str && insert_str != insert_buf)
1721 xfree((
void*)insert_str);
1725 if (insert_str != str && insert_str != insert_buf)
1726 xfree((
void*)insert_str);
1735 if (ec->replacement_allocated) {
1736 xfree((
void *)ec->replacement_str);
1738 for (i = 0; i < ec->num_trans; i++) {
1739 rb_transcoding_close(ec->elems[i].tc);
1740 xfree(ec->elems[i].out_buf_start);
1742 xfree(ec->in_buf_start);
1753 if (ec->replacement_allocated) {
1754 size += ec->replacement_len;
1756 for (i = 0; i < ec->num_trans; i++) {
1757 size += rb_transcoding_memsize(ec->elems[i].tc);
1759 if (ec->elems[i].out_buf_start) {
1760 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1763 size += ec->in_buf_end - ec->in_buf_start;
1772 if (ec->num_trans == 0)
1774 #if SIZEOF_SIZE_T > SIZEOF_INT
1775 if (ec->elems[0].tc->readagain_len > INT_MAX)
return INT_MAX;
1777 return (
int)ec->elems[0].tc->readagain_len;
1784 if (ec->num_trans == 0 || n == 0)
1786 tc = ec->elems[0].tc;
1787 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1788 tc->readagain_len -= n;
1792 const char *ascii_compat_name;
1793 const char *ascii_incompat_name;
1797 asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1803 if (DECORATOR_P(entry->sname, entry->dname))
1805 tr = load_transcoder_entry(entry);
1806 if (
tr &&
tr->asciicompat_type == asciicompat_decoder) {
1807 data->ascii_compat_name =
tr->dst_encoding;
1820 if (!st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v))
1831 if (table2->num_entries != 1)
1834 data.ascii_incompat_name = ascii_incompat_name;
1835 data.ascii_compat_name = NULL;
1836 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1837 return data.ascii_compat_name;
1849 unsigned const char *sp, *se;
1850 unsigned char *ds, *dp, *de;
1869 max_output = ec->last_tc->transcoder->max_output;
1877 unsigned long new_capa = (
unsigned long)dlen +
len + max_output;
1878 if (LONG_MAX < new_capa)
1882 sp = (
const unsigned char *)ss;
1888 switch (coderange) {
1891 cr = (int)coderange;
1900 len -= (
const char *)sp - ss;
1901 ss = (
const char *)sp;
1937 rb_econv_add_converter(
rb_econv_t *ec,
const char *sname,
const char *dname,
int n)
1942 if (ec->started != 0)
1945 entry = get_transcoder_entry(sname, dname);
1949 tr = load_transcoder_entry(entry);
1952 return rb_econv_add_transcoder_at(ec,
tr, n);
1956 rb_econv_decorate_at(
rb_econv_t *ec,
const char *decorator_name,
int n)
1958 return rb_econv_add_converter(ec,
"", decorator_name, n);
1966 if (ec->num_trans == 0)
1967 return rb_econv_decorate_at(ec, decorator_name, 0);
1969 tr = ec->elems[0].tc->transcoder;
1971 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1972 tr->asciicompat_type == asciicompat_decoder)
1973 return rb_econv_decorate_at(ec, decorator_name, 1);
1975 return rb_econv_decorate_at(ec, decorator_name, 0);
1983 if (ec->num_trans == 0)
1984 return rb_econv_decorate_at(ec, decorator_name, 0);
1986 tr = ec->elems[ec->num_trans-1].tc->transcoder;
1988 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1989 tr->asciicompat_type == asciicompat_encoder)
1990 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
1992 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
1998 const char *dname = 0;
2002 dname =
"universal_newline";
2005 dname =
"crlf_newline";
2008 dname =
"cr_newline";
2011 dname =
"lf_newline";
2016 const rb_transcoder *transcoder = get_transcoder_entry(
"", dname)->transcoder;
2017 int num_trans = ec->num_trans;
2020 for (i=0; i < num_trans; i++) {
2021 if (transcoder == ec->elems[i].tc->transcoder) {
2022 rb_transcoding_close(ec->elems[i].tc);
2023 xfree(ec->elems[i].out_buf_start);
2027 ec->elems[j++] = ec->elems[i];
2035 econv_description(
const char *sname,
const char *dname,
int ecflags,
VALUE mesg)
2037 int has_description = 0;
2042 if (*sname !=
'\0' || *dname !=
'\0') {
2045 else if (*dname ==
'\0')
2049 has_description = 1;
2056 const char *pre =
"";
2057 if (has_description)
2087 has_description = 1;
2089 if (!has_description) {
2101 econv_description(sname, dname, ecflags, mesg);
2103 exc =
rb_exc_new3(rb_eConverterNotFoundError, mesg);
2113 const char *err = (
const char *)ec->last_error.error_bytes_start;
2114 size_t error_len = ec->last_error.error_bytes_len;
2117 size_t readagain_len = ec->last_error.readagain_len;
2123 ec->last_error.source_encoding);
2125 else if (readagain_len) {
2126 bytes2 =
rb_str_new(err+error_len, readagain_len);
2131 ec->last_error.source_encoding);
2136 ec->last_error.source_encoding);
2139 exc =
rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2146 VALUE bytes =
rb_str_new((
const char *)ec->last_error.error_bytes_start,
2147 ec->last_error.error_bytes_len);
2150 if (strcmp(ec->last_error.source_encoding,
"UTF-8") == 0) {
2152 const char *start, *end;
2154 start = (
const char *)ec->last_error.error_bytes_start;
2155 end = start + ec->last_error.error_bytes_len;
2165 if (strcmp(ec->last_error.source_encoding,
2166 ec->source_encoding_name) == 0 &&
2167 strcmp(ec->last_error.destination_encoding,
2168 ec->destination_encoding_name) == 0) {
2171 ec->last_error.source_encoding,
2172 ec->last_error.destination_encoding);
2176 mesg =
rb_sprintf(
"%s to %s in conversion from %s",
2178 ec->last_error.destination_encoding,
2179 ec->source_encoding_name);
2180 for (i = 0; i < ec->num_trans; i++) {
2182 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
2184 ec->elems[i].tc->transcoder->dst_encoding);
2187 exc =
rb_exc_new3(rb_eUndefinedConversionError, mesg);
2211 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2213 unsigned char **out_start_ptr,
2214 unsigned char **out_pos,
2215 unsigned char **out_stop_ptr)
2217 size_t len = (*out_pos - *out_start_ptr);
2218 size_t new_len = (
len + max_output) * 2;
2219 *out_start_ptr = resize_destination(destination,
len, new_len);
2220 *out_pos = *out_start_ptr +
len;
2221 *out_stop_ptr = *out_start_ptr + new_len;
2229 const unsigned char *replacement;
2230 const char *repl_enc;
2231 const char *ins_enc;
2234 if (ec->replacement_str)
2241 tr = tc->transcoder;
2243 replacement = (
const unsigned char *)get_replacement_character(ins_enc, &
len, &repl_enc);
2246 replacement = (
unsigned char *)
"?";
2251 ec->replacement_str = replacement;
2252 ec->replacement_len =
len;
2253 ec->replacement_enc = repl_enc;
2254 ec->replacement_allocated = 0;
2260 const unsigned char *str,
size_t len,
const char *encname)
2262 unsigned char *str2;
2264 const char *encname2;
2268 if (!*encname2 || encoding_equal(encname, encname2)) {
2275 str2 = allocate_converted_string(encname, encname2, str,
len, NULL, 0, &len2);
2280 if (ec->replacement_allocated) {
2281 xfree((
void *)ec->replacement_str);
2283 ec->replacement_allocated = 1;
2284 ec->replacement_str = str2;
2285 ec->replacement_len = len2;
2286 ec->replacement_enc = encname2;
2295 if (make_replacement(ec) == -1)
2306 #define hash_fallback rb_hash_aref
2327 transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2328 const unsigned char *in_stop,
unsigned char *out_stop,
2330 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2331 const char *src_encoding,
2332 const char *dst_encoding,
2339 unsigned char *out_start = *out_pos;
2352 fallback_func = hash_fallback;
2355 fallback_func = proc_fallback;
2358 fallback_func = method_fallback;
2361 fallback_func = aref_fallback;
2364 last_tc = ec->last_tc;
2365 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2372 (
const char *)ec->last_error.error_bytes_start,
2373 ec->last_error.error_bytes_len,
2375 rep = (*fallback_func)(fallback, rep);
2376 if (!UNDEF_P(rep) && !
NIL_P(rep)) {
2380 if ((
int)ret == -1) {
2390 exc = make_econv_exception(ec);
2396 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2406 transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2407 const unsigned char *in_stop,
unsigned char *out_stop,
2409 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2410 const char *src_encoding,
2411 const char *dst_encoding,
2418 unsigned char *out_start = *out_pos;
2419 const unsigned char *
ptr;
2427 last_tc = ec->last_tc;
2428 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2433 unsigned char input_byte;
2434 const unsigned char *p = &input_byte;
2437 if (
ptr < in_stop) {
2448 if (&input_byte != p)
2449 ptr += p - &input_byte;
2454 exc = make_econv_exception(ec);
2460 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2481 static unsigned char *
2482 str_transcoding_resize(
VALUE destination,
size_t len,
size_t new_len)
2489 econv_opts(
VALUE opt,
int ecflags)
2492 int newlineflag = 0;
2497 else if (v==sym_replace) {
2507 else if (v==sym_replace) {
2524 else if (v==sym_attr) {
2535 #ifdef ENABLE_ECONV_NEWLINE_OPTION
2540 if (v == sym_universal) {
2543 else if (v == sym_crlf) {
2546 else if (v == sym_cr) {
2549 else if (v == sym_lf) {
2567 newlineflag |= !
NIL_P(v);
2572 newlineflag |= !
NIL_P(v);
2577 newlineflag |= !
NIL_P(v);
2582 newlineflag |= !
NIL_P(v);
2584 switch (newlineflag) {
2587 ecflags |= setflags;
2591 rb_warning(
":newline option precedes other newline options");
2605 if (
NIL_P(opthash)) {
2609 ecflags = econv_opts(opthash, ecflags);
2614 if (is_broken_string(v)) {
2637 if (!
NIL_P(newhash))
2656 if (
NIL_P(opthash)) {
2661 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2665 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2669 if (!
NIL_P(replacement)) {
2715 const char *sname, *dname;
2716 int sencidx, dencidx;
2718 dencidx = enc_arg(arg1, &dname, &denc);
2726 sencidx = enc_arg(arg2, &sname, &senc);
2737 str_transcode0(
int argc,
VALUE *argv,
VALUE *
self,
int ecflags,
VALUE ecopts)
2743 unsigned char *buf, *bp, *sp;
2744 const unsigned char *fromp;
2746 const char *sname, *dname;
2748 int explicitly_invalid_replace = TRUE;
2755 if (!ecflags)
return -1;
2759 explicitly_invalid_replace = FALSE;
2766 arg2 = argc<=1 ?
Qnil : argv[1];
2767 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2773 if (senc && senc == denc) {
2776 if (!
NIL_P(ecopts)) {
2779 dest = rb_enc_str_scrub(senc, str, rep);
2780 if (
NIL_P(dest)) dest = str;
2784 return NIL_P(arg2) ? -1 : dencidx;
2787 if (is_ascii_string(str)) {
2791 if (encoding_equal(sname, dname)) {
2792 return NIL_P(arg2) ? -1 : dencidx;
2802 if (encoding_equal(sname, dname)) {
2814 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
2815 if (fromp != sp+slen) {
2834 str_transcode(
int argc,
VALUE *argv,
VALUE *
self)
2840 argc =
rb_scan_args(argc, argv,
"02:", NULL, NULL, &opt);
2844 return str_transcode0(argc, argv,
self, ecflags, ecopts);
2848 str_encode_associate(
VALUE str,
int encidx)
2875 str_encode_bang(
int argc,
VALUE *argv,
VALUE str)
2880 rb_check_frozen(str);
2883 encidx = str_transcode(argc, argv, &newstr);
2885 if (encidx < 0)
return str;
2886 if (newstr == str) {
2891 return str_encode_associate(str, encidx);
2909 int encidx = str_transcode(argc, argv, &newstr);
2910 return encoded_dup(newstr, str, encidx);
2919 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2920 return encoded_dup(newstr, str, encidx);
2924 encoded_dup(
VALUE newstr,
VALUE str,
int encidx)
2927 if (newstr == str) {
2935 return str_encode_associate(newstr, encidx);
2944 econv_free(
void *
ptr)
2951 econv_memsize(
const void *
ptr)
2958 {0, econv_free, econv_memsize,},
2959 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2963 econv_s_allocate(
VALUE klass)
2969 make_dummy_encoding(
const char *name)
2979 make_encoding(
const char *name)
2984 enc = make_dummy_encoding(name);
2989 make_encobj(
const char *name)
3013 econv_s_asciicompat_encoding(
VALUE klass,
VALUE arg)
3015 const char *arg_name, *result_name;
3018 enc_arg(&arg, &arg_name, &arg_enc);
3022 if (result_name == NULL)
3025 result_enc = make_encoding(result_name);
3031 econv_args(
int argc,
VALUE *argv,
3033 const char **sname_p,
const char **dname_p,
3038 VALUE opt, flags_v, ecopts;
3040 const char *sname, *dname;
3044 argc =
rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
3046 if (!
NIL_P(flags_v)) {
3048 rb_error_arity(argc + 1, 2, 3);
3053 else if (!
NIL_P(opt)) {
3086 *ecflags_p = ecflags;
3091 decorate_convpath(
VALUE convpath,
int ecflags)
3094 const char *decorators[MAX_ECFLAGS_DECORATORS];
3098 num_decorators = decorator_names(ecflags, decorators);
3099 if (num_decorators == -1)
3112 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
3113 tr->asciicompat_type == asciicompat_encoder) {
3123 for (i = 0; i < num_decorators; i++)
3130 search_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3135 if (
NIL_P(*ary_p)) {
3139 if (DECORATOR_P(sname, dname)) {
3143 v =
rb_assoc_new(make_encobj(sname), make_encobj(dname));
3174 econv_s_search_convpath(
int argc,
VALUE *argv,
VALUE klass)
3176 VALUE snamev, dnamev;
3177 const char *sname, *dname;
3183 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3186 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3188 if (
NIL_P(convpath)) {
3195 if (decorate_convpath(convpath, ecflags) == -1) {
3214 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3216 return RTEST(convpath);
3226 rb_econv_init_by_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3234 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3241 rb_econv_init_by_convpath(
VALUE self,
VALUE convpath,
3242 const char **sname_p,
const char **dname_p,
3250 const char *sname, *dname;
3256 VALUE snamev, dnamev;
3263 enc_arg(&snamev, &sname, &senc);
3265 enc_arg(&dnamev, &dname, &denc);
3271 if (DECORATOR_P(sname, dname)) {
3272 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3281 int j = ec->num_trans;
3284 arg.index = ec->num_trans;
3286 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3287 if (ret == -1 || arg.ret == -1) {
3288 VALUE msg =
rb_sprintf(
"adding conversion failed: %s to %s", sname, dname);
3296 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3299 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3310 ec->source_encoding_name = *sname_p;
3311 ec->destination_encoding_name = *dname_p;
3428 econv_init(
int argc,
VALUE *argv,
VALUE self)
3431 VALUE snamev, dnamev;
3432 const char *sname, *dname;
3443 ec = rb_econv_init_by_convpath(
self, convpath, &sname, &dname, &senc, &denc);
3448 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3459 if (!DECORATOR_P(sname, dname)) {
3461 senc = make_dummy_encoding(sname);
3463 denc = make_dummy_encoding(dname);
3468 ec->source_encoding = senc;
3469 ec->destination_encoding = denc;
3487 econv_inspect(
VALUE self)
3494 return rb_sprintf(
"#<%s: uninitialized>", cname);
3496 const char *sname = ec->source_encoding_name;
3497 const char *dname = ec->destination_encoding_name;
3500 econv_description(sname, dname, ec->flags, str);
3507 check_econv(
VALUE self)
3533 econv_source_encoding(
VALUE self)
3536 return econv_get_encoding(ec->source_encoding);
3546 econv_destination_encoding(
VALUE self)
3549 return econv_get_encoding(ec->destination_encoding);
3575 econv_convpath(
VALUE self)
3582 for (i = 0; i < ec->num_trans; i++) {
3585 if (DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
3588 v =
rb_assoc_new(make_encobj(
tr->src_encoding), make_encobj(
tr->dst_encoding));
3610 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3611 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3613 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3614 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3616 if (ec1->flags != ec2->flags)
return Qfalse;
3617 if (ec1->replacement_enc != ec2->replacement_enc &&
3618 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3620 if (ec1->replacement_len != ec2->replacement_len)
return Qfalse;
3621 if (ec1->replacement_str != ec2->replacement_str &&
3622 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3625 if (ec1->num_trans != ec2->num_trans)
return Qfalse;
3626 for (i = 0; i < ec1->num_trans; i++) {
3627 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3743 econv_primitive_convert(
int argc,
VALUE *argv,
VALUE self)
3745 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3748 const unsigned char *ip, *is;
3749 unsigned char *op, *os;
3750 long output_byteoffset, output_bytesize;
3751 unsigned long output_byteend;
3754 argc =
rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3756 if (
NIL_P(output_byteoffset_v))
3757 output_byteoffset = 0;
3759 output_byteoffset =
NUM2LONG(output_byteoffset_v);
3761 if (
NIL_P(output_bytesize_v))
3762 output_bytesize = 0;
3764 output_bytesize =
NUM2LONG(output_bytesize_v);
3766 if (!
NIL_P(flags_v)) {
3768 rb_error_arity(argc + 1, 2, 5);
3772 else if (!
NIL_P(opt)) {
3791 if (
NIL_P(output_bytesize_v)) {
3800 if (
NIL_P(output_byteoffset_v))
3803 if (output_byteoffset < 0)
3809 if (output_bytesize < 0)
3812 output_byteend = (
unsigned long)output_byteoffset +
3813 (
unsigned long)output_bytesize;
3815 if (output_byteend < (
unsigned long)output_byteoffset ||
3816 LONG_MAX < output_byteend)
3830 op = (
unsigned char *)
RSTRING_PTR(output) + output_byteoffset;
3831 os = op + output_bytesize;
3835 if (!
NIL_P(input)) {
3840 if (LONG_MAX / 2 < output_bytesize)
3842 output_bytesize *= 2;
3843 output_byteoffset_v =
Qnil;
3847 if (ec->destination_encoding) {
3851 return econv_result_to_symbol(res);
3889 econv_convert(
VALUE self,
VALUE source_string)
3907 ret = econv_primitive_convert(ac, av,
self);
3909 if (ret == sym_invalid_byte_sequence ||
3910 ret == sym_undefined_conversion ||
3911 ret == sym_incomplete_input) {
3912 VALUE exc = make_econv_exception(ec);
3916 if (ret == sym_finished) {
3920 if (ret != sym_source_buffer_empty) {
3921 rb_bug(
"unexpected result of econv_primitive_convert");
3939 econv_finish(
VALUE self)
3955 ret = econv_primitive_convert(ac, av,
self);
3957 if (ret == sym_invalid_byte_sequence ||
3958 ret == sym_undefined_conversion ||
3959 ret == sym_incomplete_input) {
3960 VALUE exc = make_econv_exception(ec);
3964 if (ret != sym_finished) {
3965 rb_bug(
"unexpected result of econv_primitive_convert");
4047 econv_primitive_errinfo(
VALUE self)
4055 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4058 if (ec->last_error.source_encoding)
4061 if (ec->last_error.destination_encoding)
4064 if (ec->last_error.error_bytes_start) {
4065 rb_ary_store(ary, 3,
rb_str_new((
const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4066 rb_ary_store(ary, 4,
rb_str_new((
const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4105 econv_insert_output(
VALUE self,
VALUE string)
4107 const char *insert_enc;
4150 econv_putback(
int argc,
VALUE *argv,
VALUE self)
4163 if (putbackable < n)
4170 if (ec->source_encoding) {
4198 econv_last_error(
VALUE self)
4203 exc = make_econv_exception(ec);
4222 econv_get_replacement(
VALUE self)
4228 ret = make_replacement(ec);
4230 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4234 return rb_enc_str_new((
const char *)ec->replacement_str, (
long)ec->replacement_len, enc);
4265 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4274 return make_econv_exception(ec);
4282 exc = make_econv_exception(ec);
4295 ecerr_source_encoding_name(
VALUE self)
4297 return rb_attr_get(
self, id_source_encoding_name);
4321 ecerr_source_encoding(
VALUE self)
4333 ecerr_destination_encoding_name(
VALUE self)
4335 return rb_attr_get(
self, id_destination_encoding_name);
4345 ecerr_destination_encoding(
VALUE self)
4347 return rb_attr_get(
self, id_destination_encoding);
4366 ecerr_error_char(
VALUE self)
4387 ecerr_error_bytes(
VALUE self)
4399 ecerr_readagain_bytes(
VALUE self)
4429 ecerr_incomplete_input(
VALUE self)
4457 Init_transcode(
void)
4459 transcoder_table = st_init_strcasetable();
4462 id_destination_encoding_name =
rb_intern_const(
"destination_encoding_name");
4491 #ifdef ENABLE_ECONV_NEWLINE_OPTION
4503 InitVM_transcode(
void)
4519 rb_define_method(rb_cEncodingConverter,
"source_encoding", econv_source_encoding, 0);
4520 rb_define_method(rb_cEncodingConverter,
"destination_encoding", econv_destination_encoding, 0);
4521 rb_define_method(rb_cEncodingConverter,
"primitive_convert", econv_primitive_convert, -1);
4524 rb_define_method(rb_cEncodingConverter,
"primitive_errinfo", econv_primitive_errinfo, 0);
4525 rb_define_method(rb_cEncodingConverter,
"insert_output", econv_insert_output, 1);
4527 rb_define_method(rb_cEncodingConverter,
"last_error", econv_last_error, 0);
4528 rb_define_method(rb_cEncodingConverter,
"replacement", econv_get_replacement, 0);
4529 rb_define_method(rb_cEncodingConverter,
"replacement=", econv_set_replacement, 1);
4621 rb_define_method(rb_eUndefinedConversionError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4622 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4623 rb_define_method(rb_eUndefinedConversionError,
"source_encoding", ecerr_source_encoding, 0);
4624 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding", ecerr_destination_encoding, 0);
4625 rb_define_method(rb_eUndefinedConversionError,
"error_char", ecerr_error_char, 0);
4627 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4628 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4629 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding", ecerr_source_encoding, 0);
4630 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding", ecerr_destination_encoding, 0);
4631 rb_define_method(rb_eInvalidByteSequenceError,
"error_bytes", ecerr_error_bytes, 0);
4632 rb_define_method(rb_eInvalidByteSequenceError,
"readagain_bytes", ecerr_readagain_bytes, 0);
4633 rb_define_method(rb_eInvalidByteSequenceError,
"incomplete_input?", ecerr_incomplete_input, 0);
ruby_coderange_type
What rb_enc_str_coderange() returns.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
void rb_define_method(VALUE klass, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a method.
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
#define rb_str_new2
Old name of rb_str_new_cstr.
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
#define REALLOC_N
Old name of RB_REALLOC_N.
#define ALLOC
Old name of RB_ALLOC.
#define xfree
Old name of ruby_xfree.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
#define xrealloc
Old name of ruby_xrealloc.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
#define rb_ary_new4
Old name of rb_ary_new_from_values.
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
#define xmalloc
Old name of ruby_xmalloc.
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
#define rb_exc_new3
Old name of rb_exc_new_str.
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
#define NUM2INT
Old name of RB_NUM2INT.
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
#define ECONV_LF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_LF_NEWLINE_DECORATOR.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
#define NUM2LONG
Old name of RB_NUM2LONG.
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
#define rb_ary_new2
Old name of rb_ary_new_capa.
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
VALUE rb_eTypeError
TypeError exception.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
VALUE rb_eArgError
ArgumentError exception.
VALUE rb_eEncodingError
EncodingError exception.
void rb_warning(const char *fmt,...)
Issues a warning.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cEncoding
Encoding class.
VALUE rb_cString
String class.
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
Queries the number of bytes of the character at the passed pointer.
int rb_enc_get_index(VALUE obj)
Queries the index of the encoding of the passed object, if any.
int rb_to_encoding_index(VALUE obj)
Obtains a encoding index from a wider range of objects (than rb_enc_find_index()).
VALUE rb_enc_associate(VALUE obj, rb_encoding *enc)
Identical to rb_enc_associate_index(), except it takes an encoding itself instead of its index.
rb_encoding * rb_enc_find(const char *name)
Identical to rb_find_encoding(), except it takes a C's string instead of Ruby's.
rb_encoding * rb_to_encoding(VALUE obj)
Identical to rb_find_encoding(), except it raises an exception instead of returning NULL.
VALUE rb_enc_from_encoding(rb_encoding *enc)
Queries the Ruby-level counterpart instance of rb_cEncoding that corresponds to the passed encoding.
static bool rb_enc_asciicompat(rb_encoding *enc)
Queries if the passed encoding is in some sense compatible with ASCII.
int rb_define_dummy_encoding(const char *name)
Creates a new "dummy" encoding.
rb_encoding * rb_utf8_encoding(void)
Queries the encoding that represents UTF-8.
rb_encoding * rb_enc_from_index(int idx)
Identical to rb_find_encoding(), except it takes an encoding index instead of a Ruby object.
VALUE rb_enc_default_internal(void)
Identical to rb_default_internal_encoding(), except it returns the Ruby-level counterpart instance of...
VALUE rb_enc_associate_index(VALUE obj, int encindex)
Identical to rb_enc_set_index(), except it additionally does contents fix-up depending on the passed ...
rb_encoding * rb_enc_get(VALUE obj)
Identical to rb_enc_get_index(), except the return type.
static OnigCodePoint rb_enc_mbc_to_codepoint(const char *p, const char *e, rb_encoding *enc)
Identical to rb_enc_codepoint(), except it assumes the passed character is not broken.
static const char * rb_enc_name(rb_encoding *enc)
Queries the (canonical) name of the passed encoding.
int rb_enc_find_index(const char *name)
Queries the index of the encoding.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
VALUE rb_enc_str_new(const char *ptr, long len, rb_encoding *enc)
Identical to rb_str_new(), except it additionally takes an encoding.
VALUE rb_obj_encoding(VALUE obj)
Identical to rb_enc_get_index(), except the return type.
long rb_str_coderange_scan_restartable(const char *str, const char *end, rb_encoding *enc, int *cr)
Scans the passed string until it finds something odd.
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
rb_econv_result_t
return value of rb_econv_convert()
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
@ econv_finished
The conversion stopped after converting everything.
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
@ econv_source_buffer_empty
The conversion stopped because there is no input.
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
struct rb_econv_t rb_econv_t
An opaque struct that represents a lowest level of encoding conversion.
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
VALUE rb_assoc_new(VALUE car, VALUE cdr)
Identical to rb_ary_new_from_values(), except it expects exactly two parameters.
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_check_hash_type(VALUE obj)
Try converting an object to its hash representation using its to_hash method, if any.
VALUE rb_hash_freeze(VALUE obj)
Just another name of rb_obj_freeze.
VALUE rb_hash_aref(VALUE hash, VALUE key)
Queries the given key in the given hash table.
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Inserts or replaces ("upsert"s) the objects into the given hash table.
VALUE rb_hash_new(void)
Creates a new, empty hash object.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
VALUE rb_str_tmp_new(long len)
Allocates a "temporary" string.
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
VALUE rb_str_cat2(VALUE, const char *)
Just another name of rb_str_cat_cstr.
VALUE rb_str_dup(VALUE str)
Duplicates a string.
void rb_str_modify(VALUE str)
Declares that the string is about to be modified.
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
VALUE rb_str_new(const char *ptr, long len)
Allocates an instance of rb_cString.
VALUE rb_str_new_cstr(const char *ptr)
Identical to rb_str_new(), except it assumes the passed pointer is a pointer to a C string.
VALUE rb_str_resize(VALUE str, long len)
Overwrites the length of the string.
void rb_str_modify_expand(VALUE str, long capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
VALUE rb_str_drop_bytes(VALUE str, long len)
Shrinks the given string for the given number of bytes.
VALUE rb_attr_get(VALUE obj, ID name)
Identical to rb_ivar_get()
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
VALUE rb_sym2str(VALUE id)
Identical to rb_id2str(), except it takes an instance of rb_cSymbol rather than an ID.
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
int off
Offset inside of ptr.
int len
Length of the buffer.
VALUE rb_sprintf(const char *fmt,...)
Ruby's extended sprintf(3).
VALUE rb_str_catf(VALUE dst, const char *fmt,...)
Identical to rb_sprintf(), except it renders the output to the specified object rather than creating ...
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define ALLOCA_N(type, n)
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
#define RARRAY_LEN
Just another name of rb_array_len.
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
#define RARRAY_AREF(a, i)
#define DATA_PTR(obj)
Convenient getter macro.
#define StringValue(v)
Ensures that the parameter object is a String.
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
#define InitVM(ext)
This macro is for internal use.
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.