From: Kirill Simonov Date: Thu, 1 Jun 2006 20:19:43 +0000 (+0000) Subject: Complete buffering and encoding code. X-Git-Tag: upstream/0.1.1~39 X-Git-Url: http://andersk.mit.edu/gitweb/libyaml.git/commitdiff_plain/2fa16060e00f4e96efa26568fa85ae10ab24d9f4 Complete buffering and encoding code. --- diff --git a/include/yaml/yaml.h b/include/yaml/yaml.h index 2f32e52..cfdeacf 100644 --- a/include/yaml/yaml.h +++ b/include/yaml/yaml.h @@ -300,6 +300,9 @@ typedef struct { /** The byte about which the problem occured. */ size_t problem_offset; + /** The problematic value (@c -1 is none). */ + int problem_value; + /** * @} */ diff --git a/src/reader.c b/src/reader.c index 5ee8c43..ac11323 100644 --- a/src/reader.c +++ b/src/reader.c @@ -12,11 +12,13 @@ */ int -yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem) +yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem, + size_t offset, int value) { parser->error = YAML_READER_ERROR; parser->problem = problem; - parser->problem_offset = parser->offset; + parser->problem_offset = offset; + parser->problem_value = value; return 0; } @@ -32,9 +34,9 @@ yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem) int yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) { - /* If the EOF flag is set, do nothing. */ + /* If the EOF flag is set and the raw buffer is empty, do nothing. */ - if (parser->eof) + if (parser->eof && !parser->raw_unread) return 1; /* Return if the buffer contains enough characters. */ @@ -71,18 +73,14 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) if (!yaml_parser_update_raw_buffer(parser)) return 0; - /* If the raw buffer is empty, it is EOF. */ - - if (!parser->raw_unread) return 1; - /* Decode the raw buffer. */ while (parser->raw_unread) { unsigned int value, value2; int incomplete = 0; - unsigned char utf8_octet; - unsigned int utf8_length; + unsigned char octet; + unsigned int width; int k, low, high; /* Decode the next character. */ @@ -113,25 +111,26 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) /* Determine the length of the UTF-8 sequence. */ - utf8_octet = parser->raw_pointer[0]; - utf8_length = ( - (utf8_octet & 0x80) == 0x00 ? 1 : - (utf8_octet & 0xE0) == 0xC0 ? 2 : - (utf8_octet & 0xF0) == 0xE0 ? 3 : - (utf8_octet & 0xF8) == 0xF0 ? 4 : 0); + octet = parser->raw_pointer[0]; + width = (octet & 0x80) == 0x00 ? 1 : + (octet & 0xE0) == 0xC0 ? 2 : + (octet & 0xF0) == 0xE0 ? 3 : + (octet & 0xF8) == 0xF0 ? 4 : 0; /* Check if the leading octet is valid. */ - if (!utf8_length) + if (!width) return yaml_parser_set_reader_error(parser, - "Invalid leading UTF-8 octet"); + "Invalid leading UTF-8 octet", + parser->offset, octet); /* Check if the raw buffer contains an incomplete character. */ - if (utf8_length > parser->raw_unread) { + if (width > parser->raw_unread) { if (parser->eof) { return yaml_parser_set_reader_error(parser, - "Incomplete UTF-8 octet sequence"); + "Incomplete UTF-8 octet sequence", + parser->offset, -1); } incomplete = 1; break; @@ -139,47 +138,45 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) /* Decode the leading octet. */ - value = ( - (utf8_octet & 0x80) == 0x00 ? utf8_octet & 0x7F : - (utf8_octet & 0xE0) == 0xC0 ? utf8_octet & 0x1F : - (utf8_octet & 0xF0) == 0xE0 ? utf8_octet & 0x0F : - (utf8_octet & 0xF8) == 0xF0 ? utf8_octet & 0x07 : 0); + value = (octet & 0x80) == 0x00 ? octet & 0x7F : + (octet & 0xE0) == 0xC0 ? octet & 0x1F : + (octet & 0xF0) == 0xE0 ? octet & 0x0F : + (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; /* Check and decode the trailing octets. */ - for (k = 1; k < utf8_length; k ++) + for (k = 1; k < width; k ++) { - utf8_octet = parser->raw_pointer[k]; + octet = parser->raw_pointer[k]; /* Check if the octet is valid. */ - if ((utf8_octet & 0xC0) != 0x80) + if ((octet & 0xC0) != 0x80) return yaml_parser_set_reader_error(parser, - "Invalid trailing UTF-8 octet"); + "Invalid trailing UTF-8 octet", + parser->offset+k, octet); /* Decode the octet. */ - value = (value << 6) + (utf8_octet & 0x3F); + value = (value << 6) + (octet & 0x3F); } /* Check the length of the sequence against the value. */ - if (!((utf8_length == 1) || - (utf8_length == 2 && value >= 0x80) || - (utf8_length == 3 && value >= 0x800) || - (utf8_length == 4 && value >= 0x10000))) + if (!((width == 1) || + (width == 2 && value >= 0x80) || + (width == 3 && value >= 0x800) || + (width == 4 && value >= 0x10000))) return yaml_parser_set_reader_error(parser, - "Invalid length of a UTF-8 sequence"); + "Invalid length of a UTF-8 sequence", + parser->offset, -1); /* Check the range of the value. */ if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF) return yaml_parser_set_reader_error(parser, - "Invalid Unicode character"); - - parser->raw_pointer += utf8_length; - parser->raw_unread -= utf8_length; - parser->offset += utf8_length; + "Invalid Unicode character", + parser->offset, value); break; @@ -220,7 +217,8 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) if (parser->raw_unread < 2) { if (parser->eof) { return yaml_parser_set_reader_error(parser, - "Incomplete UTF-16 character"); + "Incomplete UTF-16 character", + parser->offset, -1); } incomplete = 1; break; @@ -235,18 +233,22 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) if ((value & 0xFC00) == 0xDC00) return yaml_parser_set_reader_error(parser, - "Unexpected low surrogate area"); + "Unexpected low surrogate area", + parser->offset, value); /* Check for a high surrogate area. */ if ((value & 0xFC00) == 0xD800) { + width = 4; + /* Check for incomplete surrogate pair. */ if (parser->raw_unread < 4) { if (parser->eof) { return yaml_parser_set_reader_error(parser, - "Incomplete UTF-16 surrogate pair"); + "Incomplete UTF-16 surrogate pair", + parser->offset, -1); } incomplete = 1; break; @@ -261,21 +263,16 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) if ((value2 & 0xFC00) != 0xDC00) return yaml_parser_set_reader_error(parser, - "Expected low surrogate area"); + "Expected low surrogate area", + parser->offset+2, value2); /* Generate the value of the surrogate pair. */ value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF); - - parser->raw_pointer += 4; - parser->raw_unread -= 4; - parser->offset += 4; } else { - parser->raw_pointer += 2; - parser->raw_unread -= 2; - parser->offset += 4; + width = 2; } break; @@ -298,7 +295,14 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) || (value >= 0xE000 && value <= 0xFFFD) || (value >= 0x10000 && value <= 0x10FFFF))) return yaml_parser_set_reader_error(parser, - "Control characters are not allowed"); + "Control characters are not allowed", + parser->offset, value); + + /* Move the raw pointers. */ + + parser->raw_pointer += width; + parser->raw_unread -= width; + parser->offset += width; /* Finally put the character into the buffer. */ @@ -309,22 +313,33 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */ else if (value <= 0x7FF) { *(parser->buffer_end++) = 0xC0 + (value >> 6); - *(parser->buffer_end++) = 0x80 + value & 0x3F; + *(parser->buffer_end++) = 0x80 + (value & 0x3F); } /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */ else if (value <= 0xFFFF) { *(parser->buffer_end++) = 0xE0 + (value >> 12); - *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F; - *(parser->buffer_end++) = 0x80 + value & 0x3F; + *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F); + *(parser->buffer_end++) = 0x80 + (value & 0x3F); } /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ else { *(parser->buffer_end++) = 0xF0 + (value >> 18); - *(parser->buffer_end++) = 0x80 + (value >> 12) & 0x3F; - *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F; - *(parser->buffer_end++) = 0x80 + value & 0x3F; + *(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F); + *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F); + *(parser->buffer_end++) = 0x80 + (value & 0x3F); } + + parser->unread ++; + } + + /* On EOF, put NUL into the buffer and return. */ + + if (parser->eof) { + *(parser->buffer_end++) = '\0'; + parser->unread ++; + return 1; } + } return 1; @@ -357,18 +372,21 @@ yaml_parser_determine_encoding(yaml_parser_t *parser) parser->encoding = YAML_UTF16LE_ENCODING; parser->raw_pointer += 2; parser->raw_unread -= 2; + parser->offset += 2; } else if (parser->raw_unread >= 2 && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) { parser->encoding = YAML_UTF16BE_ENCODING; parser->raw_pointer += 2; parser->raw_unread -= 2; + parser->offset += 2; } else if (parser->raw_unread >= 3 && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) { parser->encoding = YAML_UTF8_ENCODING; parser->raw_pointer += 3; parser->raw_unread -= 3; + parser->offset += 3; } else { parser->encoding = YAML_UTF8_ENCODING; @@ -407,7 +425,8 @@ yaml_parser_update_raw_buffer(yaml_parser_t *parser) parser->raw_buffer + parser->raw_unread, YAML_RAW_BUFFER_SIZE - parser->raw_unread, &size_read)) { - return yaml_parser_set_reader_error(parser, "Input error"); + return yaml_parser_set_reader_error(parser, "Input error", + parser->offset, -1); } parser->raw_unread += size_read; if (!size_read) { diff --git a/tests/test-reader.c b/tests/test-reader.c index b1a7426..53e4e7a 100644 --- a/tests/test-reader.c +++ b/tests/test-reader.c @@ -16,84 +16,96 @@ typedef struct { } test_case; test_case utf8_sequences[] = { - /* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */ - - {"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1}, - {"an empty line", "!", 1}, - - {"u-0 is a control character", "\x00!", 0}, - {"u-80 is a control character", "\xc2\x80!", 0}, - {"u-800 is valid", "\xe0\xa0\x80!", 1}, - {"u-10000 is valid", "\xf0\x90\x80\x80!", 1}, - {"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0}, - {"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0}, - - {"u-7f is a control character", "\x7f!", 0}, - {"u-7FF is valid", "\xdf\xbf!", 1}, - {"u-FFFF is a control character", "\xef\xbf\xbf!", 0}, - {"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0}, - {"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0}, - {"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0}, - - {"u-D7FF", "\xed\x9f\xbf!", 1}, - {"u-E000", "\xee\x80\x80!", 1}, - {"u-FFFD", "\xef\xbf\xbd!", 1}, - {"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1}, - {"u-110000", "\xf4\x90\x80\x80!", 0}, - - {"first continuation byte", "\x80!", 0}, - {"last continuation byte", "\xbf!", 0}, - - {"2 continuation bytes", "\x80\xbf!", 0}, - {"3 continuation bytes", "\x80\xbf\x80!", 0}, - {"4 continuation bytes", "\x80\xbf\x80\xbf!", 0}, - {"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0}, - {"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0}, - {"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0}, - - {"sequence of all 64 possible continuation bytes", - "\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|" - "\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|" - "\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|" - "\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0}, - {"32 first bytes of 2-byte sequences {0xc0-0xdf}", - "\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |" - "\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0}, - {"16 first bytes of 3-byte sequences {0xe0-0xef}", - "\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0}, - {"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0}, - {"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0}, - {"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0}, - - {"sequences with last byte missing {u-0}", - "\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0}, - {"sequences with last byte missing {u-...FF}", - "\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0}, - - {"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0}, - - {"overlong sequences {u-2f}", - "\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0}, - - {"maximum overlong sequences", - "\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0}, - - {"overlong representation of the NUL character", - "\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0}, - - {"single UTF-16 surrogates", - "\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0}, - - {"paired UTF-16 surrogates", - "\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|" - "\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|" - "\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0}, - - {"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0}, - - {NULL, NULL, 0} + /* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */ + + {"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1}, + {"an empty line", "!", 1}, + + {"u-0 is a control character", "\x00!", 0}, + {"u-80 is a control character", "\xc2\x80!", 0}, + {"u-800 is valid", "\xe0\xa0\x80!", 1}, + {"u-10000 is valid", "\xf0\x90\x80\x80!", 1}, + {"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0}, + {"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0}, + + {"u-7f is a control character", "\x7f!", 0}, + {"u-7FF is valid", "\xdf\xbf!", 1}, + {"u-FFFF is a control character", "\xef\xbf\xbf!", 0}, + {"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0}, + {"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0}, + {"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0}, + + {"u-D7FF", "\xed\x9f\xbf!", 1}, + {"u-E000", "\xee\x80\x80!", 1}, + {"u-FFFD", "\xef\xbf\xbd!", 1}, + {"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1}, + {"u-110000", "\xf4\x90\x80\x80!", 0}, + + {"first continuation byte", "\x80!", 0}, + {"last continuation byte", "\xbf!", 0}, + + {"2 continuation bytes", "\x80\xbf!", 0}, + {"3 continuation bytes", "\x80\xbf\x80!", 0}, + {"4 continuation bytes", "\x80\xbf\x80\xbf!", 0}, + {"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0}, + {"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0}, + {"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0}, + + {"sequence of all 64 possible continuation bytes", + "\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|" + "\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|" + "\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|" + "\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0}, + {"32 first bytes of 2-byte sequences {0xc0-0xdf}", + "\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |" + "\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0}, + {"16 first bytes of 3-byte sequences {0xe0-0xef}", + "\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0}, + {"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0}, + {"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0}, + {"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0}, + + {"sequences with last byte missing {u-0}", + "\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0}, + {"sequences with last byte missing {u-...FF}", + "\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0}, + + {"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0}, + + {"overlong sequences {u-2f}", + "\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0}, + + {"maximum overlong sequences", + "\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0}, + + {"overlong representation of the NUL character", + "\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0}, + + {"single UTF-16 surrogates", + "\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0}, + + {"paired UTF-16 surrogates", + "\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|" + "\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|" + "\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0}, + + {"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0}, + + {NULL, NULL, 0} +}; + +test_case boms[] = { + + /* {"title", "test!", lenth}, */ + + {"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, + {"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, + {"bom (utf-16-le)", "\xff\xfeH\x00i\x00 \x00i\x00s\x00 \x00\x1f\x04@\x04""8\x04""2\x04""5\x04""B\x04!", 13}, + {"bom (utf-16-be)", "\xfe\xff\x00H\x00i\x00 \x00i\x00s\x00 \x04\x1f\x04@\x04""8\x04""2\x04""5\x04""B!", 13} }; +char *bom_original = "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82"; + int check_utf8_sequences(void) { yaml_parser_t *parser; @@ -124,7 +136,14 @@ int check_utf8_sequences(void) printf("(no error)\n"); } else if (parser->error == YAML_READER_ERROR) { - printf("(reader error: %s at %d)\n", parser->problem, parser->problem_offset); + if (parser->problem_value != -1) { + printf("(reader error: %s: #%X at %d)\n", + parser->problem, parser->problem_value, parser->problem_offset); + } + else { + printf("(reader error: %s at %d)\n", + parser->problem, parser->problem_offset); + } } if (*end == '!') break; start = ++end; @@ -136,9 +155,196 @@ int check_utf8_sequences(void) return failed; } +int check_boms(void) +{ + yaml_parser_t *parser; + int failed = 0; + int k; + printf("checking boms...\n"); + for (k = 0; boms[k].test; k++) { + char *title = boms[k].title; + int check = boms[k].result; + int result; + char *start = boms[k].test; + char *end = start; + while (*end != '!') end++; + printf("\t%s: ", title); + parser = yaml_parser_new(); + assert(parser); + yaml_parser_set_input_string(parser, (unsigned char *)start, end-start); + result = yaml_parser_update_buffer(parser, end-start); + if (!result) { + printf("- (reader error: %s at %d)\n", parser->problem, parser->problem_offset); + failed++; + } + else { + if (parser->unread != check) { + printf("- (length=%d while expected length=%d)\n", parser->unread, check); + failed++; + } + else if (memcmp(parser->buffer, bom_original, check) != 0) { + printf("- (value '%s' does not equal to the original value '%s')\n", parser->buffer, bom_original); + failed++; + } + else { + printf("+\n"); + } + } + yaml_parser_delete(parser); + } + printf("checking boms: %d fail(s)\n", failed); + return failed; +} + +#define LONG 100000 + +int check_long_utf8(void) +{ + yaml_parser_t *parser; + int k = 0; + int j; + int failed = 0; + unsigned char ch0, ch1; + unsigned char *buffer = malloc(3+LONG*2); + assert(buffer); + printf("checking a long utf8 sequence...\n"); + buffer[k++] = '\xef'; + buffer[k++] = '\xbb'; + buffer[k++] = '\xbf'; + for (j = 0; j < LONG; j ++) { + if (j % 2) { + buffer[k++] = '\xd0'; + buffer[k++] = '\x90'; + } + else { + buffer[k++] = '\xd0'; + buffer[k++] = '\xaf'; + } + } + parser = yaml_parser_new(); + assert(parser); + yaml_parser_set_input_string(parser, buffer, 3+LONG*2); + for (k = 0; k < LONG; k++) { + if (!parser->unread) { + if (!yaml_parser_update_buffer(parser, 1)) { + printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset); + failed = 1; + break; + } + } + if (!parser->unread) { + printf("\tnot enough characters at %d\n", k); + failed = 1; + break; + } + if (k % 2) { + ch0 = '\xd0'; + ch1 = '\x90'; + } + else { + ch0 = '\xd0'; + ch1 = '\xaf'; + } + if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) { + printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n", + (int)parser->pointer[0], (int)parser->pointer[1], + (int)ch0, (int)ch1); + failed = 1; + break; + } + parser->pointer += 2; + parser->unread -= 1; + } + if (!failed) { + if (!yaml_parser_update_buffer(parser, 1)) { + printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset); + failed = 1; + } + else if (parser->pointer[0] != '\0') { + printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread); + failed = 1; + } + } + yaml_parser_delete(parser); + free(buffer); + printf("checking a long utf8 sequence: %d fail(s)\n", failed); + return failed; +} + +int check_long_utf16(void) +{ + yaml_parser_t *parser; + int k = 0; + int j; + int failed = 0; + unsigned char ch0, ch1; + unsigned char *buffer = malloc(2+LONG*2); + assert(buffer); + printf("checking a long utf16 sequence...\n"); + buffer[k++] = '\xff'; + buffer[k++] = '\xfe'; + for (j = 0; j < LONG; j ++) { + if (j % 2) { + buffer[k++] = '\x10'; + buffer[k++] = '\x04'; + } + else { + buffer[k++] = '/'; + buffer[k++] = '\x04'; + } + } + parser = yaml_parser_new(); + assert(parser); + yaml_parser_set_input_string(parser, buffer, 2+LONG*2); + for (k = 0; k < LONG; k++) { + if (!parser->unread) { + if (!yaml_parser_update_buffer(parser, 1)) { + printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset); + failed = 1; + break; + } + } + if (!parser->unread) { + printf("\tnot enough characters at %d\n", k); + failed = 1; + break; + } + if (k % 2) { + ch0 = '\xd0'; + ch1 = '\x90'; + } + else { + ch0 = '\xd0'; + ch1 = '\xaf'; + } + if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) { + printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n", + (int)parser->pointer[0], (int)parser->pointer[1], + (int)ch0, (int)ch1); + failed = 1; + break; + } + parser->pointer += 2; + parser->unread -= 1; + } + if (!failed) { + if (!yaml_parser_update_buffer(parser, 1)) { + printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset); + failed = 1; + } + else if (parser->pointer[0] != '\0') { + printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread); + failed = 1; + } + } + yaml_parser_delete(parser); + free(buffer); + printf("checking a long utf16 sequence: %d fail(s)\n", failed); + return failed; +} int main(void) { - return check_utf8_sequences(); + return check_utf8_sequences() + check_boms() + check_long_utf8() + check_long_utf16(); }