8 * Test cases are stolen from
9 * http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
18 test_case utf8_sequences[] = {
19 /* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */
21 {"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1},
22 {"an empty line", "!", 1},
24 {"u-0 is a control character", "\x00!", 0},
25 {"u-80 is a control character", "\xc2\x80!", 0},
26 {"u-800 is valid", "\xe0\xa0\x80!", 1},
27 {"u-10000 is valid", "\xf0\x90\x80\x80!", 1},
28 {"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0},
29 {"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0},
31 {"u-7f is a control character", "\x7f!", 0},
32 {"u-7FF is valid", "\xdf\xbf!", 1},
33 {"u-FFFF is a control character", "\xef\xbf\xbf!", 0},
34 {"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0},
35 {"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0},
36 {"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0},
38 {"u-D7FF", "\xed\x9f\xbf!", 1},
39 {"u-E000", "\xee\x80\x80!", 1},
40 {"u-FFFD", "\xef\xbf\xbd!", 1},
41 {"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1},
42 {"u-110000", "\xf4\x90\x80\x80!", 0},
44 {"first continuation byte", "\x80!", 0},
45 {"last continuation byte", "\xbf!", 0},
47 {"2 continuation bytes", "\x80\xbf!", 0},
48 {"3 continuation bytes", "\x80\xbf\x80!", 0},
49 {"4 continuation bytes", "\x80\xbf\x80\xbf!", 0},
50 {"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0},
51 {"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0},
52 {"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0},
54 {"sequence of all 64 possible continuation bytes",
55 "\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|"
56 "\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|"
57 "\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|"
58 "\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0},
59 {"32 first bytes of 2-byte sequences {0xc0-0xdf}",
60 "\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |"
61 "\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0},
62 {"16 first bytes of 3-byte sequences {0xe0-0xef}",
63 "\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0},
64 {"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0},
65 {"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0},
66 {"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0},
68 {"sequences with last byte missing {u-0}",
69 "\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0},
70 {"sequences with last byte missing {u-...FF}",
71 "\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0},
73 {"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0},
75 {"overlong sequences {u-2f}",
76 "\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0},
78 {"maximum overlong sequences",
79 "\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0},
81 {"overlong representation of the NUL character",
82 "\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0},
84 {"single UTF-16 surrogates",
85 "\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0},
87 {"paired UTF-16 surrogates",
88 "\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|"
89 "\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|"
90 "\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0},
92 {"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0},
99 /* {"title", "test!", lenth}, */
101 {"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13},
102 {"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13},
103 {"bom (utf-16-le)", "\xff\xfeH\x00i\x00 \x00i\x00s\x00 \x00\x1f\x04@\x04""8\x04""2\x04""5\x04""B\x04!", 13},
104 {"bom (utf-16-be)", "\xfe\xff\x00H\x00i\x00 \x00i\x00s\x00 \x04\x1f\x04@\x04""8\x04""2\x04""5\x04""B!", 13}
107 char *bom_original = "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82";
109 int check_utf8_sequences(void)
111 yaml_parser_t *parser;
114 printf("checking utf-8 sequences...\n");
115 for (k = 0; utf8_sequences[k].test; k++) {
116 char *title = utf8_sequences[k].title;
117 int check = utf8_sequences[k].result;
119 char *start = utf8_sequences[k].test;
121 printf("\t%s:\n", title);
123 while (*end != '|' && *end != '!') end++;
124 parser = yaml_parser_new();
126 yaml_parser_set_input_string(parser, (unsigned char *)start, end-start);
127 result = yaml_parser_update_buffer(parser, end-start);
128 if (result != check) {
135 if (!parser->error) {
136 printf("(no error)\n");
138 else if (parser->error == YAML_READER_ERROR) {
139 if (parser->problem_value != -1) {
140 printf("(reader error: %s: #%X at %d)\n",
141 parser->problem, parser->problem_value, parser->problem_offset);
144 printf("(reader error: %s at %d)\n",
145 parser->problem, parser->problem_offset);
148 if (*end == '!') break;
150 yaml_parser_delete(parser);
154 printf("checking utf-8 sequences: %d fail(s)\n", failed);
160 yaml_parser_t *parser;
163 printf("checking boms...\n");
164 for (k = 0; boms[k].test; k++) {
165 char *title = boms[k].title;
166 int check = boms[k].result;
168 char *start = boms[k].test;
170 while (*end != '!') end++;
171 printf("\t%s: ", title);
172 parser = yaml_parser_new();
174 yaml_parser_set_input_string(parser, (unsigned char *)start, end-start);
175 result = yaml_parser_update_buffer(parser, end-start);
177 printf("- (reader error: %s at %d)\n", parser->problem, parser->problem_offset);
181 if (parser->unread != check) {
182 printf("- (length=%d while expected length=%d)\n", parser->unread, check);
185 else if (memcmp(parser->buffer, bom_original, check) != 0) {
186 printf("- (value '%s' does not equal to the original value '%s')\n", parser->buffer, bom_original);
193 yaml_parser_delete(parser);
195 printf("checking boms: %d fail(s)\n", failed);
201 int check_long_utf8(void)
203 yaml_parser_t *parser;
207 unsigned char ch0, ch1;
208 unsigned char *buffer = malloc(3+LONG*2);
210 printf("checking a long utf8 sequence...\n");
211 buffer[k++] = '\xef';
212 buffer[k++] = '\xbb';
213 buffer[k++] = '\xbf';
214 for (j = 0; j < LONG; j ++) {
216 buffer[k++] = '\xd0';
217 buffer[k++] = '\x90';
220 buffer[k++] = '\xd0';
221 buffer[k++] = '\xaf';
224 parser = yaml_parser_new();
226 yaml_parser_set_input_string(parser, buffer, 3+LONG*2);
227 for (k = 0; k < LONG; k++) {
228 if (!parser->unread) {
229 if (!yaml_parser_update_buffer(parser, 1)) {
230 printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
235 if (!parser->unread) {
236 printf("\tnot enough characters at %d\n", k);
248 if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) {
249 printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n",
250 (int)parser->pointer[0], (int)parser->pointer[1],
255 parser->pointer += 2;
259 if (!yaml_parser_update_buffer(parser, 1)) {
260 printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
263 else if (parser->pointer[0] != '\0') {
264 printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread);
268 yaml_parser_delete(parser);
270 printf("checking a long utf8 sequence: %d fail(s)\n", failed);
274 int check_long_utf16(void)
276 yaml_parser_t *parser;
280 unsigned char ch0, ch1;
281 unsigned char *buffer = malloc(2+LONG*2);
283 printf("checking a long utf16 sequence...\n");
284 buffer[k++] = '\xff';
285 buffer[k++] = '\xfe';
286 for (j = 0; j < LONG; j ++) {
288 buffer[k++] = '\x10';
289 buffer[k++] = '\x04';
293 buffer[k++] = '\x04';
296 parser = yaml_parser_new();
298 yaml_parser_set_input_string(parser, buffer, 2+LONG*2);
299 for (k = 0; k < LONG; k++) {
300 if (!parser->unread) {
301 if (!yaml_parser_update_buffer(parser, 1)) {
302 printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
307 if (!parser->unread) {
308 printf("\tnot enough characters at %d\n", k);
320 if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) {
321 printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n",
322 (int)parser->pointer[0], (int)parser->pointer[1],
327 parser->pointer += 2;
331 if (!yaml_parser_update_buffer(parser, 1)) {
332 printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
335 else if (parser->pointer[0] != '\0') {
336 printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread);
340 yaml_parser_delete(parser);
342 printf("checking a long utf16 sequence: %d fail(s)\n", failed);
349 return check_utf8_sequences() + check_boms() + check_long_utf8() + check_long_utf16();