11 * Set the reader error and return 0.
15 yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem)
17 parser->error = YAML_READER_ERROR;
18 parser->problem = problem;
19 parser->problem_offset = parser->offset;
26 * Ensure that the buffer contains at least length characters.
27 * Return 1 on success, 0 on failure.
29 * The length is supposed to be significantly less that the buffer size.
33 yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
35 /* If the EOF flag is set, do nothing. */
40 /* Return if the buffer contains enough characters. */
42 if (parser->unread >= length)
45 /* Determine the input encoding if it is not known yet. */
47 if (!parser->encoding) {
48 if (!yaml_parser_determine_encoding(parser))
52 /* Move the unread characters to the beginning of the buffer. */
54 if (parser->buffer < parser->pointer
55 && parser->pointer < parser->buffer_end) {
56 size_t size = parser->buffer_end - parser->pointer;
57 memmove(parser->buffer, parser->pointer, size);
58 parser->pointer = parser->buffer;
59 parser->buffer_end -= size;
61 else if (parser->pointer == parser->buffer_end) {
62 parser->pointer = parser->buffer;
63 parser->buffer_end = parser->buffer;
66 /* Fill the buffer until it has enough characters. */
68 while (parser->unread < length)
70 /* Fill the raw buffer. */
72 if (!yaml_parser_update_raw_buffer(parser)) return 0;
74 /* If the raw buffer is empty, it is EOF. */
76 if (!parser->raw_unread) return 1;
78 /* Decode the raw buffer. */
80 while (parser->raw_unread)
82 unsigned int value, value2;
84 unsigned char utf8_octet;
85 unsigned int utf8_length;
88 /* Decode the next character. */
90 switch (parser->encoding)
92 case YAML_UTF8_ENCODING:
95 * Decode a UTF-8 character. Check RFC 3629
96 * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
98 * The following table (taken from the RFC) is used for
101 * Char. number range | UTF-8 octet sequence
102 * (hexadecimal) | (binary)
103 * --------------------+------------------------------------
104 * 0000 0000-0000 007F | 0xxxxxxx
105 * 0000 0080-0000 07FF | 110xxxxx 10xxxxxx
106 * 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
107 * 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
109 * Additionally, the characters in the range 0xD800-0xDFFF
110 * are prohibited as they are reserved for use with UTF-16
114 /* Determine the length of the UTF-8 sequence. */
116 utf8_octet = parser->raw_pointer[0];
118 (utf8_octet & 0x80) == 0x00 ? 1 :
119 (utf8_octet & 0xE0) == 0xC0 ? 2 :
120 (utf8_octet & 0xF0) == 0xE0 ? 3 :
121 (utf8_octet & 0xF8) == 0xF0 ? 4 : 0);
123 /* Check if the leading octet is valid. */
126 return yaml_parser_set_reader_error(parser,
127 "Invalid leading UTF-8 octet");
129 /* Check if the raw buffer contains an incomplete character. */
131 if (utf8_length > parser->raw_unread) {
133 return yaml_parser_set_reader_error(parser,
134 "Incomplete UTF-8 octet sequence");
140 /* Decode the leading octet. */
143 (utf8_octet & 0x80) == 0x00 ? utf8_octet & 0x7F :
144 (utf8_octet & 0xE0) == 0xC0 ? utf8_octet & 0x1F :
145 (utf8_octet & 0xF0) == 0xE0 ? utf8_octet & 0x0F :
146 (utf8_octet & 0xF8) == 0xF0 ? utf8_octet & 0x07 : 0);
148 /* Check and decode the trailing octets. */
150 for (k = 1; k < utf8_length; k ++)
152 utf8_octet = parser->raw_pointer[k];
154 /* Check if the octet is valid. */
156 if ((utf8_octet & 0xC0) != 0x80)
157 return yaml_parser_set_reader_error(parser,
158 "Invalid trailing UTF-8 octet");
160 /* Decode the octet. */
162 value = (value << 6) + (utf8_octet & 0x3F);
165 /* Check the length of the sequence against the value. */
167 if (!((utf8_length == 1) ||
168 (utf8_length == 2 && value >= 0x80) ||
169 (utf8_length == 3 && value >= 0x800) ||
170 (utf8_length == 4 && value >= 0x10000)))
171 return yaml_parser_set_reader_error(parser,
172 "Invalid length of a UTF-8 sequence");
174 /* Check the range of the value. */
176 if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
177 return yaml_parser_set_reader_error(parser,
178 "Invalid Unicode character");
180 parser->raw_pointer += utf8_length;
181 parser->raw_unread -= utf8_length;
182 parser->offset += utf8_length;
186 case YAML_UTF16LE_ENCODING:
187 case YAML_UTF16BE_ENCODING:
189 low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
190 high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);
193 * The UTF-16 encoding is not as simple as one might
194 * naively think. Check RFC 2781
195 * (http://www.ietf.org/rfc/rfc2781.txt).
197 * Normally, two subsequent bytes describe a Unicode
198 * character. However a special technique (called a
199 * surrogate pair) is used for specifying character
200 * values larger than 0xFFFF.
202 * A surrogate pair consists of two pseudo-characters:
203 * high surrogate area (0xD800-0xDBFF)
204 * low surrogate area (0xDC00-0xDFFF)
206 * The following formulas are used for decoding
207 * and encoding characters using surrogate pairs:
209 * U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
210 * U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
211 * W1 = 110110yyyyyyyyyy
212 * W2 = 110111xxxxxxxxxx
214 * where U is the character value, W1 is the high surrogate
215 * area, W2 is the low surrogate area.
218 /* Check for incomplete UTF-16 character. */
220 if (parser->raw_unread < 2) {
222 return yaml_parser_set_reader_error(parser,
223 "Incomplete UTF-16 character");
229 /* Get the character. */
231 value = parser->raw_pointer[low]
232 + (parser->raw_pointer[high] << 8);
234 /* Check for unexpected low surrogate area. */
236 if ((value & 0xFC00) == 0xDC00)
237 return yaml_parser_set_reader_error(parser,
238 "Unexpected low surrogate area");
240 /* Check for a high surrogate area. */
242 if ((value & 0xFC00) == 0xD800) {
244 /* Check for incomplete surrogate pair. */
246 if (parser->raw_unread < 4) {
248 return yaml_parser_set_reader_error(parser,
249 "Incomplete UTF-16 surrogate pair");
255 /* Get the next character. */
257 unsigned int value2 = parser->raw_pointer[low+2]
258 + (parser->raw_pointer[high+2] << 8);
260 /* Check for a low surrogate area. */
262 if ((value2 & 0xFC00) != 0xDC00)
263 return yaml_parser_set_reader_error(parser,
264 "Expected low surrogate area");
266 /* Generate the value of the surrogate pair. */
268 value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
270 parser->raw_pointer += 4;
271 parser->raw_unread -= 4;
276 parser->raw_pointer += 2;
277 parser->raw_unread -= 2;
284 /* Check if the raw buffer contains enough bytes to form a character. */
286 if (incomplete) break;
289 * Check if the character is in the allowed range:
290 * #x9 | #xA | #xD | [#x20-#x7E] (8 bit)
291 * | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit)
292 * | [#x10000-#x10FFFF] (32 bit)
295 if (! (value == 0x09 || value == 0x0A || value == 0x0D
296 || (value >= 0x20 && value <= 0x7E)
297 || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF)
298 || (value >= 0xE000 && value <= 0xFFFD)
299 || (value >= 0x10000 && value <= 0x10FFFF)))
300 return yaml_parser_set_reader_error(parser,
301 "Control characters are not allowed");
303 /* Finally put the character into the buffer. */
305 /* 0000 0000-0000 007F -> 0xxxxxxx */
307 *(parser->buffer_end++) = value;
309 /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
310 else if (value <= 0x7FF) {
311 *(parser->buffer_end++) = 0xC0 + (value >> 6);
312 *(parser->buffer_end++) = 0x80 + value & 0x3F;
314 /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
315 else if (value <= 0xFFFF) {
316 *(parser->buffer_end++) = 0xE0 + (value >> 12);
317 *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
318 *(parser->buffer_end++) = 0x80 + value & 0x3F;
320 /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
322 *(parser->buffer_end++) = 0xF0 + (value >> 18);
323 *(parser->buffer_end++) = 0x80 + (value >> 12) & 0x3F;
324 *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
325 *(parser->buffer_end++) = 0x80 + value & 0x3F;
334 * Determine the input stream encoding by checking the BOM symbol. If no BOM is
335 * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
338 #define BOM_UTF8 "\xef\xbb\xbf"
339 #define BOM_UTF16LE "\xff\xfe"
340 #define BOM_UTF16BE "\xfe\xff"
343 yaml_parser_determine_encoding(yaml_parser_t *parser)
345 /* Ensure that we had enough bytes in the raw buffer. */
347 while (!parser->eof && parser->raw_unread < 3) {
348 if (!yaml_parser_update_raw_buffer(parser)) {
353 /* Determine the encoding. */
355 if (parser->raw_unread >= 2
356 && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
357 parser->encoding = YAML_UTF16LE_ENCODING;
358 parser->raw_pointer += 2;
359 parser->raw_unread -= 2;
361 else if (parser->raw_unread >= 2
362 && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
363 parser->encoding = YAML_UTF16BE_ENCODING;
364 parser->raw_pointer += 2;
365 parser->raw_unread -= 2;
367 else if (parser->raw_unread >= 3
368 && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
369 parser->encoding = YAML_UTF8_ENCODING;
370 parser->raw_pointer += 3;
371 parser->raw_unread -= 3;
374 parser->encoding = YAML_UTF8_ENCODING;
381 * Update the raw buffer.
385 yaml_parser_update_raw_buffer(yaml_parser_t *parser)
387 size_t size_read = 0;
389 /* Return if the raw buffer is full. */
391 if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;
395 if (parser->eof) return 1;
397 /* Move the remaining bytes in the raw buffer to the beginning. */
399 if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
400 memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
402 parser->raw_pointer = parser->raw_buffer;
404 /* Call the read handler to fill the buffer. */
406 if (!parser->read_handler(parser->read_handler_data,
407 parser->raw_buffer + parser->raw_unread,
408 YAML_RAW_BUFFER_SIZE - parser->raw_unread,
410 return yaml_parser_set_reader_error(parser, "Input error");
412 parser->raw_unread += size_read;