12#include "ruby/internal/config.h"
17#include "internal/array.h"
18#include "internal/inits.h"
19#include "internal/object.h"
20#include "internal/string.h"
21#include "internal/transcode.h"
22#include "internal/encoding.h"
26#include "transcode_data.h"
29#define ENABLE_ECONV_NEWLINE_OPTION 1
32static VALUE rb_eUndefinedConversionError;
33static VALUE rb_eInvalidByteSequenceError;
34static VALUE rb_eConverterNotFoundError;
36VALUE rb_cEncodingConverter;
38static ID id_destination_encoding;
39static ID id_destination_encoding_name;
40static ID id_error_bytes;
41static ID id_error_char;
42static ID id_incomplete_input;
43static ID id_readagain_bytes;
44static ID id_source_encoding;
45static ID id_source_encoding_name;
47static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
48static VALUE sym_xml, sym_text, sym_attr;
49static VALUE sym_universal_newline;
50static VALUE sym_crlf_newline;
51static VALUE sym_cr_newline;
52static VALUE sym_lf_newline;
53#ifdef ENABLE_ECONV_NEWLINE_OPTION
54static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
56static VALUE sym_partial_input;
58static VALUE sym_invalid_byte_sequence;
59static VALUE sym_undefined_conversion;
60static VALUE sym_destination_buffer_full;
61static VALUE sym_source_buffer_empty;
62static VALUE sym_finished;
63static VALUE sym_after_output;
64static VALUE sym_incomplete_input;
67allocate_converted_string(
const char *sname,
const char *dname,
68 const unsigned char *str,
size_t len,
69 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
80 unsigned int next_table;
82 unsigned char next_byte;
83 unsigned int output_index;
85 ssize_t recognized_len;
86 ssize_t readagain_len;
101 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
102 double dummy_for_alignment;
105#define TRANSCODING_READBUF(tc) \
106 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
107 (tc)->readbuf.ary : \
109#define TRANSCODING_WRITEBUF(tc) \
110 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
111 (tc)->writebuf.ary : \
113#define TRANSCODING_WRITEBUF_SIZE(tc) \
114 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
115 sizeof((tc)->writebuf.ary) : \
116 (size_t)(tc)->transcoder->max_output)
117#define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
118#define TRANSCODING_STATE(tc) \
119 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
125 unsigned char *out_buf_start;
126 unsigned char *out_data_start;
127 unsigned char *out_data_end;
128 unsigned char *out_buf_end;
136 const char *source_encoding_name;
137 const char *destination_encoding_name;
139 const unsigned char *replacement_str;
140 size_t replacement_len;
141 const char *replacement_enc;
143 unsigned char *in_buf_start;
144 unsigned char *in_data_start;
145 unsigned char *in_data_end;
146 unsigned char *in_buf_end;
148 int replacement_allocated;
158 const char *source_encoding;
159 const char *destination_encoding;
160 const unsigned char *error_bytes_start;
161 size_t error_bytes_len;
162 size_t readagain_len;
175#define DECORATOR_P(sname, dname) (*(sname) == '\0')
187free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
194free_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
196 st_foreach((
void *)val, free_inner_transcode_i, 0);
197 st_free_table((
void *)val);
202rb_free_transcoder_table(
void)
204 st_foreach(transcoder_table, free_transcode_i, 0);
205 st_free_table(transcoder_table);
209make_transcoder_entry(
const char *sname,
const char *dname)
215 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
216 val = (st_data_t)st_init_strcasetable();
217 st_add_direct(transcoder_table, (st_data_t)sname, val);
220 if (!st_lookup(table2, (st_data_t)dname, &val)) {
222 entry->sname = sname;
223 entry->dname = dname;
225 entry->transcoder = NULL;
226 val = (st_data_t)entry;
227 st_add_direct(table2, (st_data_t)dname, val);
234get_transcoder_entry(
const char *sname,
const char *dname)
239 if (st_lookup(transcoder_table, (st_data_t)sname, &val)) {
241 if (!st_lookup(table2, (st_data_t)dname, &val)) {
252 const char *
const sname =
tr->src_encoding;
253 const char *
const dname =
tr->dst_encoding;
258 entry = make_transcoder_entry(sname, dname);
259 if (entry->transcoder) {
260 rb_raise(rb_eArgError,
"transcoder from %s to %s has been already registered",
263 entry->transcoder =
tr;
268declare_transcoder(
const char *sname,
const char *dname,
const char *lib)
272 entry = make_transcoder_entry(sname, dname);
276static const char transcoder_lib_prefix[] =
"enc/trans/";
279rb_declare_transcoder(
const char *enc1,
const char *enc2,
const char *lib)
282 rb_raise(rb_eArgError,
"invalid library name - (null)");
284 declare_transcoder(enc1, enc2, lib);
287#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
298 const char *base_enc;
302transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
304 const char *dname = (
const char *)key;
308 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
315 *bfs->queue_last_ptr = q;
316 bfs->queue_last_ptr = &q->next;
318 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
323transcode_search_path(
const char *sname,
const char *dname,
324 void (*callback)(
const char *sname,
const char *dname,
int depth,
void *arg),
335 if (encoding_equal(sname, dname))
341 bfs.queue_last_ptr = &q->next;
344 bfs.visited = st_init_strcasetable();
345 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
352 bfs.queue_last_ptr = &bfs.queue;
355 lookup_res = st_lookup(transcoder_table, (st_data_t)q->enc, &val);
362 if (st_lookup(table2, (st_data_t)dname, &val)) {
363 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
369 bfs.base_enc = q->enc;
370 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
384 const char *enc = dname;
388 st_lookup(bfs.visited, (st_data_t)enc, &val);
392 enc = (
const char *)val;
397 st_lookup(bfs.visited, (st_data_t)enc, &val);
400 callback((
const char *)val, enc, --depth, arg);
401 enc = (
const char *)val;
405 st_free_table(bfs.visited);
410int rb_require_internal_silent(
VALUE fname);
415 ASSERT_vm_unlocking();
416 if (entry->transcoder)
417 return entry->transcoder;
420 const char *
const lib = entry->lib;
421 const size_t len = strlen(lib);
422 const size_t total_len =
sizeof(transcoder_lib_prefix) - 1 +
len;
424 char *
const path = RSTRING_PTR(fn);
426 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
427 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib,
len);
430 rb_require_internal_silent(fn);
433 if (entry->transcoder)
434 return entry->transcoder;
440get_replacement_character(
const char *encname,
size_t *len_ret,
const char **repl_encname_ptr)
442 if (encoding_equal(encname,
"UTF-8")) {
444 *repl_encname_ptr =
"UTF-8";
445 return "\xEF\xBF\xBD";
449 *repl_encname_ptr =
"US-ASCII";
458static const unsigned char *
460 const unsigned char *in_start,
461 const unsigned char *inchar_start,
462 const unsigned char *in_p,
463 size_t *char_len_ptr)
465 const unsigned char *ptr;
466 if (inchar_start - in_start < tc->recognized_len) {
467 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
468 inchar_start,
unsigned char, in_p - inchar_start);
469 ptr = TRANSCODING_READBUF(tc);
472 ptr = inchar_start - tc->recognized_len;
474 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
479transcode_restartable0(
const unsigned char **in_pos,
unsigned char **out_pos,
480 const unsigned char *in_stop,
unsigned char *out_stop,
485 int unitlen =
tr->input_unit_length;
486 ssize_t readagain_len = 0;
488 const unsigned char *inchar_start;
489 const unsigned char *in_p;
491 unsigned char *out_p;
493 in_p = inchar_start = *in_pos;
497#define SUSPEND(ret, num) \
499 tc->resume_position = (num); \
500 if (0 < in_p - inchar_start) \
501 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
502 inchar_start, unsigned char, in_p - inchar_start); \
505 tc->recognized_len += in_p - inchar_start; \
506 if (readagain_len) { \
507 tc->recognized_len -= readagain_len; \
508 tc->readagain_len = readagain_len; \
511 resume_label ## num:; \
513#define SUSPEND_OBUF(num) \
515 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
518#define SUSPEND_AFTER_OUTPUT(num) \
519 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
520 SUSPEND(econv_after_output, num); \
523#define next_table (tc->next_table)
524#define next_info (tc->next_info)
525#define next_byte (tc->next_byte)
526#define writebuf_len (tc->writebuf_len)
527#define writebuf_off (tc->writebuf_off)
529 switch (tc->resume_position) {
531 case 1:
goto resume_label1;
532 case 2:
goto resume_label2;
533 case 3:
goto resume_label3;
534 case 4:
goto resume_label4;
535 case 5:
goto resume_label5;
536 case 6:
goto resume_label6;
537 case 7:
goto resume_label7;
538 case 8:
goto resume_label8;
539 case 9:
goto resume_label9;
540 case 10:
goto resume_label10;
541 case 11:
goto resume_label11;
542 case 12:
goto resume_label12;
543 case 13:
goto resume_label13;
544 case 14:
goto resume_label14;
545 case 15:
goto resume_label15;
546 case 16:
goto resume_label16;
547 case 17:
goto resume_label17;
548 case 18:
goto resume_label18;
549 case 19:
goto resume_label19;
550 case 20:
goto resume_label20;
551 case 21:
goto resume_label21;
552 case 22:
goto resume_label22;
553 case 23:
goto resume_label23;
554 case 24:
goto resume_label24;
555 case 25:
goto resume_label25;
556 case 26:
goto resume_label26;
557 case 27:
goto resume_label27;
558 case 28:
goto resume_label28;
559 case 29:
goto resume_label29;
560 case 30:
goto resume_label30;
561 case 31:
goto resume_label31;
562 case 32:
goto resume_label32;
563 case 33:
goto resume_label33;
564 case 34:
goto resume_label34;
569 tc->recognized_len = 0;
570 next_table =
tr->conv_tree_start;
572 SUSPEND_AFTER_OUTPUT(24);
574 if (in_stop <= in_p) {
581#define BYTE_ADDR(index) (tr->byte_array + (index))
582#define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
583#define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
584#define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
585#define BL_MIN_BYTE (BL_BASE[0])
586#define BL_MAX_BYTE (BL_BASE[1])
587#define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
588#define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
590 next_byte = (
unsigned char)*in_p++;
592 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
595 next_info = (
VALUE)BL_ACTION(next_byte);
598 switch (next_info & 0x1F) {
601 const unsigned char *p = inchar_start;
604 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (
unsigned char)*p++;
606 writebuf_len = writebuf_off;
608 while (writebuf_off < writebuf_len) {
610 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
614 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
615 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
616 SUSPEND_AFTER_OUTPUT(25);
617 while (in_p >= in_stop) {
622 next_byte = (
unsigned char)*in_p++;
623 next_table = (
unsigned int)next_info;
628 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
631 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
632 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
635 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
636 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
637 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
640 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
641 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
642 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
643 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
646 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
647 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
648 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
649 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
652 tc->output_index = 0;
653 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
654 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
659 next_info = (
VALUE)(*
tr->func_ii)(TRANSCODING_STATE(tc), next_info);
663 const unsigned char *char_start;
665 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
666 next_info = (
VALUE)(*
tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
671 if (
tr->max_output <= out_stop - out_p)
672 out_p +=
tr->func_io(TRANSCODING_STATE(tc),
673 next_info, out_p, out_stop - out_p);
675 writebuf_len =
tr->func_io(TRANSCODING_STATE(tc),
677 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
679 while (writebuf_off < writebuf_len) {
681 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
687 const unsigned char *char_start;
690 if (
tr->max_output <= out_stop - out_p) {
691 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
692 out_p +=
tr->func_so(TRANSCODING_STATE(tc),
693 char_start, (
size_t)char_len,
694 out_p, out_stop - out_p);
697 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
698 writebuf_len =
tr->func_so(TRANSCODING_STATE(tc),
699 char_start, (
size_t)char_len,
700 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
702 while (writebuf_off < writebuf_len) {
704 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
711 const unsigned char *char_start;
714 if (
tr->max_output <= out_stop - out_p) {
715 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
716 out_p +=
tr->func_sio(TRANSCODING_STATE(tc),
717 char_start, (
size_t)char_len, next_info,
718 out_p, out_stop - out_p);
721 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
722 writebuf_len =
tr->func_sio(TRANSCODING_STATE(tc),
723 char_start, (
size_t)char_len, next_info,
724 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
726 while (writebuf_off < writebuf_len) {
728 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
734 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
735 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
736 SUSPEND_AFTER_OUTPUT(26);
737 while ((opt &
ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
741 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
745 in_p = inchar_start + (unitlen - tc->recognized_len);
751 invalid_len = tc->recognized_len + (in_p - inchar_start);
752 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
753 readagain_len = invalid_len - discard_len;
777 if (
tr->finish_func) {
779 if (
tr->max_output <= out_stop - out_p) {
780 out_p +=
tr->finish_func(TRANSCODING_STATE(tc),
781 out_p, out_stop - out_p);
784 writebuf_len =
tr->finish_func(TRANSCODING_STATE(tc),
785 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
787 while (writebuf_off < writebuf_len) {
789 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
804transcode_restartable(
const unsigned char **in_pos,
unsigned char **out_pos,
805 const unsigned char *in_stop,
unsigned char *out_stop,
809 if (tc->readagain_len) {
810 unsigned char *readagain_buf =
ALLOCA_N(
unsigned char, tc->readagain_len);
811 const unsigned char *readagain_pos = readagain_buf;
812 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
815 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
816 unsigned char, tc->readagain_len);
817 tc->readagain_len = 0;
818 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|
ECONV_PARTIAL_INPUT);
820 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
821 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
822 tc->readagain_len += readagain_stop - readagain_pos;
826 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
837 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
839 if (
tr->state_init_func) {
840 (
tr->state_init_func)(TRANSCODING_STATE(tc));
842 tc->resume_position = 0;
843 tc->recognized_len = 0;
844 tc->readagain_len = 0;
845 tc->writebuf_len = 0;
846 tc->writebuf_off = 0;
847 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
848 tc->readbuf.ptr =
xmalloc(
tr->max_input);
850 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
851 tc->writebuf.ptr =
xmalloc(
tr->max_output);
858 const unsigned char **input_ptr,
const unsigned char *input_stop,
859 unsigned char **output_ptr,
unsigned char *output_stop,
862 return transcode_restartable(
863 input_ptr, output_ptr,
864 input_stop, output_stop,
872 if (
tr->state_fini_func) {
873 (
tr->state_fini_func)(TRANSCODING_STATE(tc));
875 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
876 xfree(tc->state.ptr);
877 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input)
878 xfree(tc->readbuf.ptr);
879 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output)
880 xfree(tc->writebuf.ptr);
890 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
891 size +=
tr->state_size;
893 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
894 size +=
tr->max_input;
896 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
897 size +=
tr->max_output;
903rb_econv_alloc(
int n_hint)
912 ec->source_encoding_name = NULL;
913 ec->destination_encoding_name = NULL;
915 ec->replacement_str = NULL;
916 ec->replacement_len = 0;
917 ec->replacement_enc = NULL;
918 ec->replacement_allocated = 0;
919 ec->in_buf_start = NULL;
920 ec->in_data_start = NULL;
921 ec->in_data_end = NULL;
922 ec->in_buf_end = NULL;
923 ec->num_allocated = n_hint;
926 ec->num_finished = 0;
929 ec->last_error.error_tc = NULL;
930 ec->last_error.source_encoding = NULL;
931 ec->last_error.destination_encoding = NULL;
932 ec->last_error.error_bytes_start = NULL;
933 ec->last_error.error_bytes_len = 0;
934 ec->last_error.readagain_len = 0;
935 ec->source_encoding = NULL;
936 ec->destination_encoding = NULL;
947 if (ec->num_trans == ec->num_allocated) {
948 n = ec->num_allocated * 2;
950 ec->num_allocated = n;
957 ec->elems[i].tc = rb_transcoding_open_by_transcoder(
tr, 0);
958 ec->elems[i].out_buf_start = p;
959 ec->elems[i].out_buf_end = p + bufsize;
960 ec->elems[i].out_data_start = p;
961 ec->elems[i].out_data_end = p;
966 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
967 for (j = ec->num_trans-1; i <= j; j--) {
970 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
985 for (i = 0; i < n; i++) {
987 tr = load_transcoder_entry(entries[i]);
992 ec = rb_econv_alloc(n);
994 for (i = 0; i < n; i++) {
996 ret = rb_econv_add_transcoder_at(ec,
tr, ec->num_trans);
1012trans_open_i(
const char *sname,
const char *dname,
int depth,
void *arg)
1016 if (!toarg->entries) {
1019 toarg->entries[depth] = get_transcoder_entry(sname, dname);
1023rb_econv_open0(
const char *sname,
const char *dname,
int ecflags)
1030 if (*sname) rb_enc_find_index(sname);
1031 if (*dname) rb_enc_find_index(dname);
1033 if (*sname ==
'\0' && *dname ==
'\0') {
1040 toarg.entries = NULL;
1041 toarg.num_additional = 0;
1042 num_trans = transcode_search_path(sname, dname, trans_open_i, (
void *)&toarg);
1043 entries = toarg.entries;
1044 if (num_trans < 0) {
1050 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1055 ec->flags = ecflags;
1056 ec->source_encoding_name = sname;
1057 ec->destination_encoding_name = dname;
1062#define MAX_ECFLAGS_DECORATORS 32
1065decorator_names(
int ecflags,
const char **decorators_ret)
1087 decorators_ret[num_decorators++] =
"xml_text_escape";
1089 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1091 decorators_ret[num_decorators++] =
"xml_attr_quote";
1094 decorators_ret[num_decorators++] =
"crlf_newline";
1096 decorators_ret[num_decorators++] =
"cr_newline";
1098 decorators_ret[num_decorators++] =
"lf_newline";
1100 decorators_ret[num_decorators++] =
"universal_newline";
1102 return num_decorators;
1110 const char *decorators[MAX_ECFLAGS_DECORATORS];
1113 num_decorators = decorator_names(ecflags, decorators);
1114 if (num_decorators == -1)
1119 for (i = 0; i < num_decorators; i++) {
1129 ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1136 const unsigned char **input_ptr,
const unsigned char *input_stop,
1137 unsigned char **output_ptr,
unsigned char *output_stop,
1144 const unsigned char **ipp, *is, *iold;
1145 unsigned char **opp, *os, *oold;
1151 for (i = start; i < ec->num_trans; i++) {
1160 ipp = (
const unsigned char **)&prev_te->out_data_start;
1161 is = prev_te->out_data_end;
1164 if (i == ec->num_trans-1) {
1169 if (te->out_buf_start != te->out_data_start) {
1170 ssize_t
len = te->out_data_end - te->out_data_start;
1171 ssize_t
off = te->out_data_start - te->out_buf_start;
1172 MEMMOVE(te->out_buf_start, te->out_data_start,
unsigned char,
len);
1173 te->out_data_start = te->out_buf_start;
1174 te->out_data_end -=
off;
1176 opp = &te->out_data_end;
1177 os = te->out_buf_end;
1181 if (ec->num_finished != i)
1185 flags &= ~ECONV_AFTER_OUTPUT;
1188 f &= ~ECONV_AFTER_OUTPUT;
1191 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1192 if (iold != *ipp || oold != *opp)
1207 ec->num_finished = i+1;
1217 const unsigned char **input_ptr,
const unsigned char *input_stop,
1218 unsigned char **output_ptr,
unsigned char *output_stop,
1220 int *result_position_ptr)
1223 int needreport_index;
1226 unsigned char empty_buf;
1227 unsigned char *empty_ptr = &empty_buf;
1230 input_ptr = (
const unsigned char **)&empty_ptr;
1231 input_stop = empty_ptr;
1235 output_ptr = &empty_ptr;
1236 output_stop = empty_ptr;
1242 for (i = ec->num_trans-1; 0 <= i; i--) {
1243 switch (ec->elems[i].last_result) {
1250 goto found_needreport;
1257 rb_bug(
"unexpected transcode last result");
1267 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1269 result_position_ptr);
1281 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1282 sweep_start = needreport_index + 1;
1283 }
while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1285 for (i = ec->num_trans-1; 0 <= i; i--) {
1294 if (result_position_ptr)
1295 *result_position_ptr = i;
1299 if (result_position_ptr)
1300 *result_position_ptr = -1;
1306 const unsigned char **input_ptr,
const unsigned char *input_stop,
1307 unsigned char **output_ptr,
unsigned char *output_stop,
1311 int result_position;
1314 memset(&ec->last_error, 0,
sizeof(ec->last_error));
1316 if (ec->num_trans == 0) {
1318 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1319 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1320 len = output_stop - *output_ptr;
1321 memcpy(*output_ptr, ec->in_data_start,
len);
1322 *output_ptr = output_stop;
1323 ec->in_data_start +=
len;
1327 len = ec->in_data_end - ec->in_data_start;
1328 memcpy(*output_ptr, ec->in_data_start,
len);
1330 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1336 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1337 len = output_stop - *output_ptr;
1340 len = input_stop - *input_ptr;
1343 *(*output_ptr)++ = *(*input_ptr)++;
1347 memcpy(*output_ptr, *input_ptr,
len);
1350 if (*input_ptr != input_stop)
1359 if (ec->elems[ec->num_trans-1].out_data_start) {
1360 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1361 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1362 if (data_start != data_end) {
1364 if (output_stop - *output_ptr < data_end - data_start) {
1365 len = output_stop - *output_ptr;
1366 memcpy(*output_ptr, data_start,
len);
1367 *output_ptr = output_stop;
1368 ec->elems[ec->num_trans-1].out_data_start +=
len;
1372 len = data_end - data_start;
1373 memcpy(*output_ptr, data_start,
len);
1375 ec->elems[ec->num_trans-1].out_data_start =
1376 ec->elems[ec->num_trans-1].out_data_end =
1377 ec->elems[ec->num_trans-1].out_buf_start;
1382 if (ec->in_buf_start &&
1383 ec->in_data_start != ec->in_data_end) {
1384 res = rb_trans_conv(ec, (
const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1392 *input_ptr != input_stop) {
1393 input_stop = *input_ptr;
1394 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1399 ec->num_trans == 1) {
1400 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1405 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1410 ec->last_error.result = res;
1415 ec->last_error.error_tc = error_tc;
1416 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1417 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1418 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1419 ec->last_error.error_bytes_len = error_tc->recognized_len;
1420 ec->last_error.readagain_len = error_tc->readagain_len;
1426static int output_replacement_character(
rb_econv_t *ec);
1432 unsigned char utfbuf[1024];
1433 const unsigned char *utf;
1435 int utf_allocated = 0;
1436 char charef_buf[16];
1437 const unsigned char *p;
1439 if (encoding_equal(ec->last_error.source_encoding,
"UTF-32BE")) {
1440 utf = ec->last_error.error_bytes_start;
1441 utf_len = ec->last_error.error_bytes_len;
1444 utf = allocate_converted_string(ec->last_error.source_encoding,
"UTF-32BE",
1445 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1446 utfbuf,
sizeof(utfbuf),
1450 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1454 if (utf_len % 4 != 0)
1458 while (4 <= utf_len) {
1464 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1486 const unsigned char **input_ptr,
const unsigned char *input_stop,
1487 unsigned char **output_ptr,
unsigned char *output_stop,
1492 unsigned char empty_buf;
1493 unsigned char *empty_ptr = &empty_buf;
1498 input_ptr = (
const unsigned char **)&empty_ptr;
1499 input_stop = empty_ptr;
1503 output_ptr = &empty_ptr;
1504 output_stop = empty_ptr;
1508 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1516 if (output_replacement_character(ec) == 0)
1527 if (output_replacement_character(ec) == 0)
1532 if (output_hex_charref(ec) == 0)
1550 tr = tc->transcoder;
1552 if (
tr->asciicompat_type == asciicompat_encoder)
1553 return tr->src_encoding;
1554 return tr->dst_encoding;
1557static unsigned char *
1558allocate_converted_string(
const char *sname,
const char *dname,
1559 const unsigned char *str,
size_t len,
1560 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1561 size_t *dst_len_ptr)
1563 unsigned char *dst_str;
1570 const unsigned char *sp;
1574 dst_bufsize = caller_dst_bufsize;
1584 dst_str = caller_dst_buf;
1586 dst_str =
xmalloc(dst_bufsize);
1589 dp = dst_str+dst_len;
1591 dst_len = dp - dst_str;
1593 if (SIZE_MAX/2 < dst_bufsize) {
1597 if (dst_str == caller_dst_buf) {
1600 memcpy(tmp, dst_str, dst_bufsize/2);
1604 dst_str =
xrealloc(dst_str, dst_bufsize);
1606 dp = dst_str+dst_len;
1608 dst_len = dp - dst_str;
1614 *dst_len_ptr = dst_len;
1618 if (dst_str != caller_dst_buf)
1627 const unsigned char *str,
size_t len,
const char *str_encoding)
1630 unsigned char insert_buf[4096];
1631 const unsigned char *insert_str = NULL;
1634 int last_trans_index;
1637 unsigned char **buf_start_p;
1638 unsigned char **data_start_p;
1639 unsigned char **data_end_p;
1640 unsigned char **buf_end_p;
1649 if (encoding_equal(insert_encoding, str_encoding)) {
1654 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1655 str,
len, insert_buf,
sizeof(insert_buf), &insert_len);
1656 if (insert_str == NULL)
1662 last_trans_index = ec->num_trans-1;
1663 if (ec->num_trans == 0) {
1665 buf_start_p = &ec->in_buf_start;
1666 data_start_p = &ec->in_data_start;
1667 data_end_p = &ec->in_data_end;
1668 buf_end_p = &ec->in_buf_end;
1670 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1671 tc = ec->elems[last_trans_index].tc;
1672 need += tc->readagain_len;
1673 if (need < insert_len)
1675 if (last_trans_index == 0) {
1676 buf_start_p = &ec->in_buf_start;
1677 data_start_p = &ec->in_data_start;
1678 data_end_p = &ec->in_data_end;
1679 buf_end_p = &ec->in_buf_end;
1683 buf_start_p = &ee->out_buf_start;
1684 data_start_p = &ee->out_data_start;
1685 data_end_p = &ee->out_data_end;
1686 buf_end_p = &ee->out_buf_end;
1691 buf_start_p = &ee->out_buf_start;
1692 data_start_p = &ee->out_data_start;
1693 data_end_p = &ee->out_data_end;
1694 buf_end_p = &ee->out_buf_end;
1695 tc = ec->elems[last_trans_index].tc;
1698 if (*buf_start_p == NULL) {
1699 unsigned char *buf =
xmalloc(need);
1701 *data_start_p = buf;
1703 *buf_end_p = buf+need;
1705 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1706 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1707 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1708 *data_start_p = *buf_start_p;
1709 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1711 size_t s = (*data_end_p - *buf_start_p) + need;
1715 *data_start_p = buf;
1716 *data_end_p = buf + (*data_end_p - *buf_start_p);
1718 *buf_end_p = buf + s;
1722 memcpy(*data_end_p, insert_str, insert_len);
1723 *data_end_p += insert_len;
1724 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1725 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1726 *data_end_p += tc->readagain_len;
1727 tc->readagain_len = 0;
1730 if (insert_str != str && insert_str != insert_buf)
1731 xfree((
void*)insert_str);
1735 if (insert_str != str && insert_str != insert_buf)
1736 xfree((
void*)insert_str);
1745 if (ec->replacement_allocated) {
1746 xfree((
void *)ec->replacement_str);
1748 for (i = 0; i < ec->num_trans; i++) {
1749 rb_transcoding_close(ec->elems[i].tc);
1750 xfree(ec->elems[i].out_buf_start);
1752 xfree(ec->in_buf_start);
1763 if (ec->replacement_allocated) {
1764 size += ec->replacement_len;
1766 for (i = 0; i < ec->num_trans; i++) {
1767 size += rb_transcoding_memsize(ec->elems[i].tc);
1769 if (ec->elems[i].out_buf_start) {
1770 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1773 size += ec->in_buf_end - ec->in_buf_start;
1782 if (ec->num_trans == 0)
1784#if SIZEOF_SIZE_T > SIZEOF_INT
1785 if (ec->elems[0].tc->readagain_len > INT_MAX)
return INT_MAX;
1787 return (
int)ec->elems[0].tc->readagain_len;
1794 if (ec->num_trans == 0 || n == 0)
1796 tc = ec->elems[0].tc;
1797 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1798 tc->readagain_len -= n;
1802 const char *ascii_compat_name;
1803 const char *ascii_incompat_name;
1807asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1813 if (DECORATOR_P(entry->sname, entry->dname))
1815 tr = load_transcoder_entry(entry);
1816 if (
tr &&
tr->asciicompat_type == asciicompat_decoder) {
1817 data->ascii_compat_name =
tr->dst_encoding;
1831 RB_VM_LOCK_ENTER_LEV(&lev);
1833 if (st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v)) {
1842 if (table2->num_entries == 1) {
1843 data.ascii_incompat_name = ascii_incompat_name;
1844 data.ascii_compat_name = NULL;
1845 if (rb_multi_ractor_p()) {
1850 st_table *dup_table2 = st_copy(table2);
1851 RB_VM_LOCK_LEAVE_LEV(&lev);
1852 st_foreach(dup_table2, asciicompat_encoding_i, (st_data_t)&data);
1853 st_free_table(dup_table2);
1854 RB_VM_LOCK_ENTER_LEV(&lev);
1857 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1863 RB_VM_LOCK_LEAVE_LEV(&lev);
1865 return data.ascii_compat_name;
1877 unsigned const char *sp, *se;
1878 unsigned char *ds, *dp, *de;
1887 rb_enc_associate(dst, dst_enc);
1892 dst_enc = rb_enc_get(dst);
1897 max_output = ec->last_tc->transcoder->max_output;
1903 long dlen = RSTRING_LEN(dst);
1905 unsigned long new_capa = (
unsigned long)dlen +
len + max_output;
1906 if (LONG_MAX < new_capa)
1907 rb_raise(rb_eArgError,
"too long string");
1910 sp = (
const unsigned char *)ss;
1912 ds = (
unsigned char *)RSTRING_PTR(dst);
1916 switch (coderange) {
1919 cr = (int)coderange;
1928 len -= (
const char *)sp - ss;
1929 ss = (
const char *)sp;
1965rb_econv_add_converter(
rb_econv_t *ec,
const char *sname,
const char *dname,
int n)
1970 if (ec->started != 0)
1973 entry = get_transcoder_entry(sname, dname);
1975 tr = load_transcoder_entry(entry);
1978 return tr ? rb_econv_add_transcoder_at(ec,
tr, n) : -1;
1982rb_econv_decorate_at(
rb_econv_t *ec,
const char *decorator_name,
int n)
1984 return rb_econv_add_converter(ec,
"", decorator_name, n);
1992 if (ec->num_trans == 0)
1993 return rb_econv_decorate_at(ec, decorator_name, 0);
1995 tr = ec->elems[0].tc->transcoder;
1997 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1998 tr->asciicompat_type == asciicompat_decoder)
1999 return rb_econv_decorate_at(ec, decorator_name, 1);
2001 return rb_econv_decorate_at(ec, decorator_name, 0);
2009 if (ec->num_trans == 0)
2010 return rb_econv_decorate_at(ec, decorator_name, 0);
2012 tr = ec->elems[ec->num_trans-1].tc->transcoder;
2014 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
2015 tr->asciicompat_type == asciicompat_encoder)
2016 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
2018 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
2024 const char *dname = 0;
2028 dname =
"universal_newline";
2031 dname =
"crlf_newline";
2034 dname =
"cr_newline";
2037 dname =
"lf_newline";
2042 const rb_transcoder *transcoder = get_transcoder_entry(
"", dname)->transcoder;
2043 int num_trans = ec->num_trans;
2046 for (i=0; i < num_trans; i++) {
2047 if (transcoder == ec->elems[i].tc->transcoder) {
2048 rb_transcoding_close(ec->elems[i].tc);
2049 xfree(ec->elems[i].out_buf_start);
2053 ec->elems[j++] = ec->elems[i];
2057 ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2061econv_description(
const char *sname,
const char *dname,
int ecflags,
VALUE mesg)
2063 int has_description = 0;
2068 if (*sname !=
'\0' || *dname !=
'\0') {
2071 else if (*dname ==
'\0')
2074 rb_str_catf(mesg,
"%s to %s", sname, dname);
2075 has_description = 1;
2082 const char *pre =
"";
2083 if (has_description)
2113 has_description = 1;
2115 if (!has_description) {
2127 econv_description(sname, dname, ecflags, mesg);
2129 exc =
rb_exc_new3(rb_eConverterNotFoundError, mesg);
2139 const char *err = (
const char *)ec->last_error.error_bytes_start;
2140 size_t error_len = ec->last_error.error_bytes_len;
2143 size_t readagain_len = ec->last_error.readagain_len;
2147 mesg = rb_sprintf(
"incomplete %s on %s",
2149 ec->last_error.source_encoding);
2151 else if (readagain_len) {
2152 bytes2 =
rb_str_new(err+error_len, readagain_len);
2154 mesg = rb_sprintf(
"%s followed by %s on %s",
2157 ec->last_error.source_encoding);
2160 mesg = rb_sprintf(
"%s on %s",
2162 ec->last_error.source_encoding);
2165 exc =
rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2172 VALUE bytes =
rb_str_new((
const char *)ec->last_error.error_bytes_start,
2173 ec->last_error.error_bytes_len);
2176 if (strcmp(ec->last_error.source_encoding,
"UTF-8") == 0) {
2178 const char *start, *end;
2180 start = (
const char *)ec->last_error.error_bytes_start;
2181 end = start + ec->last_error.error_bytes_len;
2182 n = rb_enc_precise_mbclen(start, end, utf8);
2185 unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
2186 dumped = rb_sprintf(
"U+%04X", cc);
2191 if (strcmp(ec->last_error.source_encoding,
2192 ec->source_encoding_name) == 0 &&
2193 strcmp(ec->last_error.destination_encoding,
2194 ec->destination_encoding_name) == 0) {
2195 mesg = rb_sprintf(
"%s from %s to %s",
2197 ec->last_error.source_encoding,
2198 ec->last_error.destination_encoding);
2202 mesg = rb_sprintf(
"%s to %s in conversion from %s",
2204 ec->last_error.destination_encoding,
2205 ec->source_encoding_name);
2206 for (i = 0; i < ec->num_trans; i++) {
2208 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
2209 rb_str_catf(mesg,
" to %s",
2210 ec->elems[i].tc->transcoder->dst_encoding);
2213 exc =
rb_exc_new3(rb_eUndefinedConversionError, mesg);
2214 idx = rb_enc_find_index(ec->last_error.source_encoding);
2216 rb_enc_associate_index(bytes, idx);
2225 int idx = rb_enc_find_index(ec->last_error.source_encoding);
2227 rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2228 idx = rb_enc_find_index(ec->last_error.destination_encoding);
2230 rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2237 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2239 unsigned char **out_start_ptr,
2240 unsigned char **out_pos,
2241 unsigned char **out_stop_ptr)
2243 size_t len = (*out_pos - *out_start_ptr);
2244 size_t new_len = (
len + max_output) * 2;
2245 *out_start_ptr = resize_destination(destination,
len, new_len);
2246 *out_pos = *out_start_ptr +
len;
2247 *out_stop_ptr = *out_start_ptr + new_len;
2255 const unsigned char *replacement;
2256 const char *repl_enc;
2257 const char *ins_enc;
2260 if (ec->replacement_str)
2267 tr = tc->transcoder;
2268 rb_enc_find(
tr->dst_encoding);
2269 replacement = (
const unsigned char *)get_replacement_character(ins_enc, &
len, &repl_enc);
2272 replacement = (
unsigned char *)
"?";
2277 ec->replacement_str = replacement;
2278 ec->replacement_len =
len;
2279 ec->replacement_enc = repl_enc;
2280 ec->replacement_allocated = 0;
2286 const unsigned char *str,
size_t len,
const char *encname)
2288 unsigned char *str2;
2290 const char *encname2;
2294 if (!*encname2 || encoding_equal(encname, encname2)) {
2301 str2 = allocate_converted_string(encname, encname2, str,
len, NULL, 0, &len2);
2306 if (ec->replacement_allocated) {
2307 xfree((
void *)ec->replacement_str);
2309 ec->replacement_allocated = 1;
2310 ec->replacement_str = str2;
2311 ec->replacement_len = len2;
2312 ec->replacement_enc = encname2;
2321 if (make_replacement(ec) == -1)
2332#define hash_fallback rb_hash_aref
2353transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2354 const unsigned char *in_stop,
unsigned char *out_stop,
2356 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2357 const char *src_encoding,
2358 const char *dst_encoding,
2365 unsigned char *out_start = *out_pos;
2376 fallback = rb_hash_aref(ecopts, sym_fallback);
2378 fallback_func = hash_fallback;
2381 fallback_func = proc_fallback;
2384 fallback_func = method_fallback;
2387 fallback_func = aref_fallback;
2390 last_tc = ec->last_tc;
2391 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2397 VALUE rep = rb_enc_str_new(
2398 (
const char *)ec->last_error.error_bytes_start,
2399 ec->last_error.error_bytes_len,
2400 rb_enc_find(ec->last_error.source_encoding));
2401 rep = (*fallback_func)(fallback, rep);
2402 if (!UNDEF_P(rep) && !
NIL_P(rep)) {
2405 RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
2406 if ((
int)ret == -1) {
2407 rb_raise(rb_eArgError,
"too big fallback string");
2416 exc = make_econv_exception(ec);
2422 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2432transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2433 const unsigned char *in_stop,
unsigned char *out_stop,
2435 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2436 const char *src_encoding,
2437 const char *dst_encoding,
2444 unsigned char *out_start = *out_pos;
2445 const unsigned char *ptr;
2453 last_tc = ec->last_tc;
2454 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2459 unsigned char input_byte;
2460 const unsigned char *p = &input_byte;
2463 if (ptr < in_stop) {
2474 if (&input_byte != p)
2475 ptr += p - &input_byte;
2480 exc = make_econv_exception(ec);
2486 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2507static unsigned char *
2508str_transcoding_resize(
VALUE destination,
size_t len,
size_t new_len)
2510 rb_str_resize(destination, new_len);
2511 return (
unsigned char *)RSTRING_PTR(destination);
2515econv_opts(
VALUE opt,
int ecflags)
2518 int newlineflag = 0;
2520 v = rb_hash_aref(opt, sym_invalid);
2523 else if (v==sym_replace) {
2527 rb_raise(rb_eArgError,
"unknown value for invalid character option");
2530 v = rb_hash_aref(opt, sym_undef);
2533 else if (v==sym_replace) {
2537 rb_raise(rb_eArgError,
"unknown value for undefined character option");
2540 v = rb_hash_aref(opt, sym_replace);
2545 v = rb_hash_aref(opt, sym_xml);
2550 else if (v==sym_attr) {
2554 rb_raise(rb_eArgError,
"unexpected value for xml option: %"PRIsVALUE,
rb_sym2str(v));
2557 rb_raise(rb_eArgError,
"unexpected value for xml option");
2561#ifdef ENABLE_ECONV_NEWLINE_OPTION
2562 v = rb_hash_aref(opt, sym_newline);
2565 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2566 if (v == sym_universal) {
2569 else if (v == sym_crlf) {
2572 else if (v == sym_cr) {
2575 else if (v == sym_lf) {
2579 rb_raise(rb_eArgError,
"unexpected value for newline option: %"PRIsVALUE,
2583 rb_raise(rb_eArgError,
"unexpected value for newline option");
2590 v = rb_hash_aref(opt, sym_universal_newline);
2593 newlineflag |= !
NIL_P(v);
2595 v = rb_hash_aref(opt, sym_crlf_newline);
2598 newlineflag |= !
NIL_P(v);
2600 v = rb_hash_aref(opt, sym_cr_newline);
2603 newlineflag |= !
NIL_P(v);
2605 v = rb_hash_aref(opt, sym_lf_newline);
2608 newlineflag |= !
NIL_P(v);
2610 switch (newlineflag) {
2612 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2613 ecflags |= setflags;
2617 rb_warning(
":newline option precedes other newline options");
2631 if (
NIL_P(opthash)) {
2635 ecflags = econv_opts(opthash, ecflags);
2637 v = rb_hash_aref(opthash, sym_replace);
2640 if (is_broken_string(v)) {
2642 rb_raise(rb_eArgError,
"replacement string is broken: %s as %s",
2644 rb_enc_name(rb_enc_get(v)));
2647 newhash = rb_hash_new();
2648 rb_hash_aset(newhash, sym_replace, v);
2651 v = rb_hash_aref(opthash, sym_fallback);
2653 VALUE h = rb_check_hash_type(v);
2658 newhash = rb_hash_new();
2659 rb_hash_aset(newhash, sym_fallback, v);
2663 if (!
NIL_P(newhash))
2664 rb_hash_freeze(newhash);
2682 if (
NIL_P(opthash)) {
2687 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2688 replacement = rb_hash_aref(opthash, sym_replace);
2691 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2693 if (!
NIL_P(replacement)) {
2698 (
const unsigned char *)RSTRING_PTR(replacement),
2699 RSTRING_LEN(replacement),
2718 if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
2719 !(enc = rb_enc_from_index(encidx))) {
2725 n = rb_enc_name(enc);
2740 const char *sname, *dname;
2741 int sencidx, dencidx;
2743 dencidx = enc_arg(arg1, &dname, &denc);
2746 sencidx = rb_enc_get_index(str);
2747 senc = rb_enc_from_index(sencidx);
2748 sname = rb_enc_name(senc);
2751 sencidx = enc_arg(arg2, &sname, &senc);
2762str_transcode0(
int argc,
VALUE *argv,
VALUE *self,
int ecflags,
VALUE ecopts)
2768 unsigned char *buf, *bp, *sp;
2769 const unsigned char *fromp;
2771 const char *sname, *dname;
2773 int explicitly_invalid_replace = TRUE;
2778 arg1 = rb_enc_default_internal();
2780 if (!ecflags)
return -1;
2781 arg1 = rb_obj_encoding(str);
2784 explicitly_invalid_replace = FALSE;
2791 arg2 = argc<=1 ?
Qnil : argv[1];
2792 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2798 if (senc && senc == denc) {
2801 if (!
NIL_P(ecopts)) {
2802 rep = rb_hash_aref(ecopts, sym_replace);
2804 dest = rb_enc_str_scrub(senc, str, rep);
2805 if (
NIL_P(dest)) dest = str;
2809 return NIL_P(arg2) ? -1 : dencidx;
2811 if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
2812 if (is_ascii_string(str)) {
2816 if (encoding_equal(sname, dname)) {
2817 return NIL_P(arg2) ? -1 : dencidx;
2821 if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) {
2827 if (encoding_equal(sname, dname)) {
2833 fromp = sp = (
unsigned char *)RSTRING_PTR(str);
2834 slen = RSTRING_LEN(str);
2837 bp = (
unsigned char *)RSTRING_PTR(dest);
2839 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
2840 if (fromp != sp+slen) {
2841 rb_raise(rb_eArgError,
"not fully converted, %"PRIdPTRDIFF
" bytes left", sp+slen-fromp);
2843 buf = (
unsigned char *)RSTRING_PTR(dest);
2849 dencidx = rb_define_dummy_encoding(dname);
2859str_transcode(
int argc,
VALUE *argv,
VALUE *self)
2865 argc =
rb_scan_args(argc, argv,
"02:", NULL, NULL, &opt);
2869 return str_transcode0(argc, argv, self, ecflags, ecopts);
2873str_encode_associate(
VALUE str,
int encidx)
2877 rb_enc_associate_index(str, encidx);
2880 if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
2901str_encode_bang(
int argc,
VALUE *argv,
VALUE str)
2906 rb_check_frozen(str);
2909 encidx = str_transcode(argc, argv, &newstr);
2911 if (encidx < 0)
return str;
2912 if (newstr == str) {
2913 rb_enc_associate_index(str, encidx);
2917 return str_encode_associate(str, encidx);
2935 int encidx = str_transcode(argc, argv, &newstr);
2936 return encoded_dup(newstr, str, encidx);
2945 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2946 return encoded_dup(newstr, str, encidx);
2950encoded_dup(
VALUE newstr,
VALUE str,
int encidx)
2953 if (newstr == str) {
2955 rb_enc_associate_index(newstr, encidx);
2961 return str_encode_associate(newstr, encidx);
2970econv_free(
void *ptr)
2977econv_memsize(
const void *ptr)
2984 {0, econv_free, econv_memsize,},
2985 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2989econv_s_allocate(
VALUE klass)
2995make_dummy_encoding(
const char *name)
2999 idx = rb_define_dummy_encoding(name);
3000 enc = rb_enc_from_index(idx);
3005make_encoding(
const char *name)
3008 enc = rb_enc_find(name);
3011 if (rb_enc_registered(name)) {
3015 enc = make_dummy_encoding(name);
3023make_encobj(
const char *name)
3025 return rb_enc_from_encoding(make_encoding(name));
3047econv_s_asciicompat_encoding(
VALUE klass,
VALUE arg)
3049 const char *arg_name, *result_name;
3053 enc_arg(&arg, &arg_name, &arg_enc);
3056 result_enc = make_encoding(result_name);
3057 enc = rb_enc_from_encoding(result_enc);
3063econv_args(
int argc,
VALUE *argv,
3065 const char **sname_p,
const char **dname_p,
3070 VALUE opt, flags_v, ecopts;
3072 const char *sname, *dname;
3076 argc =
rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
3078 if (!
NIL_P(flags_v)) {
3080 rb_error_arity(argc + 1, 2, 3);
3085 else if (!
NIL_P(opt)) {
3094 sidx = rb_to_encoding_index(*snamev_p);
3096 senc = rb_enc_from_index(sidx);
3103 didx = rb_to_encoding_index(*dnamev_p);
3105 denc = rb_enc_from_index(didx);
3118 *ecflags_p = ecflags;
3123decorate_convpath(
VALUE convpath,
int ecflags)
3126 const char *decorators[MAX_ECFLAGS_DECORATORS];
3130 num_decorators = decorator_names(ecflags, decorators);
3131 if (num_decorators == -1)
3138 const char *sname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 0)));
3139 const char *dname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 1)));
3142 entry = get_transcoder_entry(sname, dname);
3143 tr = load_transcoder_entry(entry);
3146 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
3147 tr->asciicompat_type == asciicompat_encoder) {
3157 for (i = 0; i < num_decorators; i++)
3164search_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3169 if (
NIL_P(*ary_p)) {
3173 if (DECORATOR_P(sname, dname)) {
3177 v =
rb_assoc_new(make_encobj(sname), make_encobj(dname));
3208econv_s_search_convpath(
int argc,
VALUE *argv,
VALUE klass)
3210 VALUE snamev, dnamev;
3211 const char *sname, *dname;
3217 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3220 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3222 if (
NIL_P(convpath)) {
3229 if (decorate_convpath(convpath, ecflags) == -1) {
3248 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3250 return RTEST(convpath);
3260rb_econv_init_by_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3268 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3275rb_econv_init_by_convpath(
VALUE self,
VALUE convpath,
3276 const char **sname_p,
const char **dname_p,
3284 const char *sname, *dname;
3290 VALUE snamev, dnamev;
3295 rb_raise(rb_eArgError,
"not a 2-element array in convpath");
3297 enc_arg(&snamev, &sname, &senc);
3299 enc_arg(&dnamev, &dname, &denc);
3305 if (DECORATOR_P(sname, dname)) {
3306 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3308 VALUE msg = rb_sprintf(
"decoration failed: %s", dname);
3315 int j = ec->num_trans;
3318 arg.index = ec->num_trans;
3320 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3321 if (ret == -1 || arg.ret == -1) {
3322 VALUE msg = rb_sprintf(
"adding conversion failed: %s to %s", sname, dname);
3330 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3333 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3344 ec->source_encoding_name = *sname_p;
3345 ec->destination_encoding_name = *dname_p;
3465 VALUE snamev, dnamev;
3466 const char *sname, *dname;
3477 ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
3482 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3493 if (!DECORATOR_P(sname, dname)) {
3495 senc = make_dummy_encoding(sname);
3497 denc = make_dummy_encoding(dname);
3502 ec->source_encoding = senc;
3503 ec->destination_encoding = denc;
3521econv_inspect(
VALUE self)
3528 return rb_sprintf(
"#<%s: uninitialized>", cname);
3530 const char *sname = ec->source_encoding_name;
3531 const char *dname = ec->destination_encoding_name;
3533 str = rb_sprintf(
"#<%s: ", cname);
3534 econv_description(sname, dname, ec->flags, str);
3541check_econv(
VALUE self)
3547 rb_raise(
rb_eTypeError,
"uninitialized encoding converter");
3557 return rb_enc_from_encoding(encoding);
3567econv_source_encoding(
VALUE self)
3570 return econv_get_encoding(ec->source_encoding);
3580econv_destination_encoding(
VALUE self)
3583 return econv_get_encoding(ec->destination_encoding);
3609econv_convpath(
VALUE self)
3616 for (i = 0; i < ec->num_trans; i++) {
3619 if (DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
3622 v =
rb_assoc_new(make_encobj(
tr->src_encoding), make_encobj(
tr->dst_encoding));
3644 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3645 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3647 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3648 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3650 if (ec1->flags != ec2->flags)
return Qfalse;
3651 if (ec1->replacement_enc != ec2->replacement_enc &&
3652 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3654 if (ec1->replacement_len != ec2->replacement_len)
return Qfalse;
3655 if (ec1->replacement_str != ec2->replacement_str &&
3656 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3659 if (ec1->num_trans != ec2->num_trans)
return Qfalse;
3660 for (i = 0; i < ec1->num_trans; i++) {
3661 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3777econv_primitive_convert(
int argc,
VALUE *argv,
VALUE self)
3779 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3782 const unsigned char *ip, *is;
3783 unsigned char *op, *os;
3784 long output_byteoffset, output_bytesize;
3785 unsigned long output_byteend;
3788 argc =
rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3790 if (
NIL_P(output_byteoffset_v))
3791 output_byteoffset = 0;
3793 output_byteoffset =
NUM2LONG(output_byteoffset_v);
3795 if (
NIL_P(output_bytesize_v))
3796 output_bytesize = 0;
3798 output_bytesize =
NUM2LONG(output_bytesize_v);
3800 if (!
NIL_P(flags_v)) {
3802 rb_error_arity(argc + 1, 2, 5);
3806 else if (!
NIL_P(opt)) {
3809 v = rb_hash_aref(opt, sym_partial_input);
3812 v = rb_hash_aref(opt, sym_after_output);
3823 rb_str_modify(output);
3825 if (
NIL_P(output_bytesize_v)) {
3828 if (!
NIL_P(input) && output_bytesize < RSTRING_LEN(input))
3829 output_bytesize = RSTRING_LEN(input);
3834 if (
NIL_P(output_byteoffset_v))
3835 output_byteoffset = RSTRING_LEN(output);
3837 if (output_byteoffset < 0)
3838 rb_raise(rb_eArgError,
"negative output_byteoffset");
3840 if (RSTRING_LEN(output) < output_byteoffset)
3841 rb_raise(rb_eArgError,
"output_byteoffset too big");
3843 if (output_bytesize < 0)
3844 rb_raise(rb_eArgError,
"negative output_bytesize");
3846 output_byteend = (
unsigned long)output_byteoffset +
3847 (
unsigned long)output_bytesize;
3849 if (output_byteend < (
unsigned long)output_byteoffset ||
3850 LONG_MAX < output_byteend)
3851 rb_raise(rb_eArgError,
"output_byteoffset+output_bytesize too big");
3854 rb_str_resize(output, output_byteend);
3860 ip = (
const unsigned char *)RSTRING_PTR(input);
3861 is = ip + RSTRING_LEN(input);
3864 op = (
unsigned char *)RSTRING_PTR(output) + output_byteoffset;
3865 os = op + output_bytesize;
3869 if (!
NIL_P(input)) {
3874 if (LONG_MAX / 2 < output_bytesize)
3875 rb_raise(rb_eArgError,
"too long conversion result");
3876 output_bytesize *= 2;
3877 output_byteoffset_v =
Qnil;
3881 if (ec->destination_encoding) {
3882 rb_enc_associate(output, ec->destination_encoding);
3885 return econv_result_to_symbol(res);
3923econv_convert(
VALUE self,
VALUE source_string)
3941 ret = econv_primitive_convert(ac, av, self);
3943 if (ret == sym_invalid_byte_sequence ||
3944 ret == sym_undefined_conversion ||
3945 ret == sym_incomplete_input) {
3946 VALUE exc = make_econv_exception(ec);
3950 if (ret == sym_finished) {
3951 rb_raise(rb_eArgError,
"converter already finished");
3954 if (ret != sym_source_buffer_empty) {
3955 rb_bug(
"unexpected result of econv_primitive_convert");
3973econv_finish(
VALUE self)
3989 ret = econv_primitive_convert(ac, av, self);
3991 if (ret == sym_invalid_byte_sequence ||
3992 ret == sym_undefined_conversion ||
3993 ret == sym_incomplete_input) {
3994 VALUE exc = make_econv_exception(ec);
3998 if (ret != sym_finished) {
3999 rb_bug(
"unexpected result of econv_primitive_convert");
4081econv_primitive_errinfo(
VALUE self)
4089 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4092 if (ec->last_error.source_encoding)
4095 if (ec->last_error.destination_encoding)
4098 if (ec->last_error.error_bytes_start) {
4099 rb_ary_store(ary, 3,
rb_str_new((
const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4100 rb_ary_store(ary, 4,
rb_str_new((
const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4141 const char *insert_enc;
4149 string =
rb_str_encode(
string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0,
Qnil);
4151 ret =
rb_econv_insert_output(ec, (
const unsigned char *)RSTRING_PTR(
string), RSTRING_LEN(
string), insert_enc);
4153 rb_raise(rb_eArgError,
"too big string");
4184econv_putback(
int argc,
VALUE *argv,
VALUE self)
4197 if (putbackable < n)
4204 if (ec->source_encoding) {
4205 rb_enc_associate(str, ec->source_encoding);
4232econv_last_error(
VALUE self)
4237 exc = make_econv_exception(ec);
4256econv_get_replacement(
VALUE self)
4262 ret = make_replacement(ec);
4264 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4267 enc = rb_enc_find(ec->replacement_enc);
4268 return rb_enc_str_new((
const char *)ec->replacement_str, (
long)ec->replacement_len, enc);
4290 enc = rb_enc_get(
string);
4293 (
const unsigned char *)RSTRING_PTR(
string),
4294 RSTRING_LEN(
string),
4299 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4308 return make_econv_exception(ec);
4316 exc = make_econv_exception(ec);
4329ecerr_source_encoding_name(
VALUE self)
4331 return rb_attr_get(self, id_source_encoding_name);
4355ecerr_source_encoding(
VALUE self)
4357 return rb_attr_get(self, id_source_encoding);
4367ecerr_destination_encoding_name(
VALUE self)
4369 return rb_attr_get(self, id_destination_encoding_name);
4379ecerr_destination_encoding(
VALUE self)
4381 return rb_attr_get(self, id_destination_encoding);
4400ecerr_error_char(
VALUE self)
4402 return rb_attr_get(self, id_error_char);
4421ecerr_error_bytes(
VALUE self)
4423 return rb_attr_get(self, id_error_bytes);
4433ecerr_readagain_bytes(
VALUE self)
4435 return rb_attr_get(self, id_readagain_bytes);
4463ecerr_incomplete_input(
VALUE self)
4465 return rb_attr_get(self, id_incomplete_input);
4493 transcoder_table = st_init_strcasetable();
4496 id_destination_encoding_name =
rb_intern_const(
"destination_encoding_name");
4522 sym_lf_newline =
ID2SYM(rb_intern(
"lf_newline"));
4525#ifdef ENABLE_ECONV_NEWLINE_OPTION
4537InitVM_transcode(
void)
4553 rb_define_method(rb_cEncodingConverter,
"source_encoding", econv_source_encoding, 0);
4554 rb_define_method(rb_cEncodingConverter,
"destination_encoding", econv_destination_encoding, 0);
4555 rb_define_method(rb_cEncodingConverter,
"primitive_convert", econv_primitive_convert, -1);
4558 rb_define_method(rb_cEncodingConverter,
"primitive_errinfo", econv_primitive_errinfo, 0);
4559 rb_define_method(rb_cEncodingConverter,
"insert_output", econv_insert_output, 1);
4561 rb_define_method(rb_cEncodingConverter,
"last_error", econv_last_error, 0);
4562 rb_define_method(rb_cEncodingConverter,
"replacement", econv_get_replacement, 0);
4563 rb_define_method(rb_cEncodingConverter,
"replacement=", econv_set_replacement, 1);
4641 rb_define_method(rb_eUndefinedConversionError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4642 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4643 rb_define_method(rb_eUndefinedConversionError,
"source_encoding", ecerr_source_encoding, 0);
4644 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding", ecerr_destination_encoding, 0);
4645 rb_define_method(rb_eUndefinedConversionError,
"error_char", ecerr_error_char, 0);
4647 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4648 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4649 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding", ecerr_source_encoding, 0);
4650 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding", ecerr_destination_encoding, 0);
4651 rb_define_method(rb_eInvalidByteSequenceError,
"error_bytes", ecerr_error_bytes, 0);
4652 rb_define_method(rb_eInvalidByteSequenceError,
"readagain_bytes", ecerr_readagain_bytes, 0);
4653 rb_define_method(rb_eInvalidByteSequenceError,
"incomplete_input?", ecerr_incomplete_input, 0);
ruby_coderange_type
What rb_enc_str_coderange() returns.
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
#define rb_str_new2
Old name of rb_str_new_cstr.
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
#define REALLOC_N
Old name of RB_REALLOC_N.
#define ALLOC
Old name of RB_ALLOC.
#define xfree
Old name of ruby_xfree.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
#define rb_str_cat2
Old name of rb_str_cat_cstr.
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
#define xrealloc
Old name of ruby_xrealloc.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
#define rb_ary_new4
Old name of rb_ary_new_from_values.
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
#define xmalloc
Old name of ruby_xmalloc.
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
#define rb_exc_new3
Old name of rb_exc_new_str.
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
#define NUM2INT
Old name of RB_NUM2INT.
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
#define ECONV_LF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_LF_NEWLINE_DECORATOR.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
#define NUM2LONG
Old name of RB_NUM2LONG.
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
#define rb_ary_new2
Old name of rb_ary_new_capa.
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
VALUE rb_eTypeError
TypeError exception.
VALUE rb_eRuntimeError
RuntimeError exception.
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
VALUE rb_eEncodingError
EncodingError exception.
void rb_warning(const char *fmt,...)
Issues a warning.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cEncoding
Encoding class.
VALUE rb_cString
String class.
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
long rb_str_coderange_scan_restartable(const char *str, const char *end, rb_encoding *enc, int *cr)
Scans the passed string until it finds something odd.
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
rb_econv_result_t
return value of rb_econv_convert()
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
@ econv_finished
The conversion stopped after converting everything.
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
@ econv_source_buffer_empty
The conversion stopped because there is no input.
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
VALUE rb_assoc_new(VALUE car, VALUE cdr)
Identical to rb_ary_new_from_values(), except it expects exactly two parameters.
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
VALUE rb_str_tmp_new(long len)
Allocates a "temporary" string.
#define rb_str_new(str, len)
Allocates an instance of rb_cString.
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
VALUE rb_str_dup(VALUE str)
Duplicates a string.
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
void rb_str_modify_expand(VALUE str, long capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
VALUE rb_str_drop_bytes(VALUE str, long len)
Shrinks the given string for the given number of bytes.
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
int off
Offset inside of ptr.
int len
Length of the buffer.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define ALLOCA_N(type, n)
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
#define RARRAY_LEN
Just another name of rb_array_len.
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
#define RARRAY_AREF(a, i)
#define DATA_PTR(obj)
Convenient getter macro.
#define StringValue(v)
Ensures that the parameter object is a String.
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
#define InitVM(ext)
This macro is for internal use.
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.