[libyaml.git] / src / reader.c


#if HAVE_CONFIG_H
#include <config.h>
#endif

#include <yaml/yaml.h>

#include <assert.h>

/*
 * Set the reader error and return 0.
 */

int
yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
        size_t offset, int value)
{
    parser->error = YAML_READER_ERROR;
    parser->problem = problem;
    parser->problem_offset = offset;
    parser->problem_value = value;

    return 0;
}


/*
 * Ensure that the buffer contains at least length characters.
 * Return 1 on success, 0 on failure.
 *
 * The length is supposed to be significantly less that the buffer size.
 */

int
yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
{
    /* If the EOF flag is set and the raw buffer is empty, do nothing. */

    if (parser->eof && !parser->raw_unread)
        return 1;

    /* Return if the buffer contains enough characters. */

    if (parser->unread >= length)
        return 1;

    /* Determine the input encoding if it is not known yet. */

    if (!parser->encoding) {
        if (!yaml_parser_determine_encoding(parser))
            return 0;
    }

    /* Move the unread characters to the beginning of the buffer. */

    if (parser->buffer < parser->pointer
            && parser->pointer < parser->buffer_end) {
        size_t size = parser->buffer_end - parser->pointer;
        memmove(parser->buffer, parser->pointer, size);
        parser->pointer = parser->buffer;
        parser->buffer_end -= size;
    }
    else if (parser->pointer == parser->buffer_end) {
        parser->pointer = parser->buffer;
        parser->buffer_end = parser->buffer;
    }

    /* Fill the buffer until it has enough characters. */

    while (parser->unread < length)
    {
        /* Fill the raw buffer. */

        if (!yaml_parser_update_raw_buffer(parser)) return 0;

        /* Decode the raw buffer. */

        while (parser->raw_unread)
        {
            unsigned int value, value2;
            int incomplete = 0;
            unsigned char octet;
            unsigned int width;
            int k, low, high;

            /* Decode the next character. */

            switch (parser->encoding)
            {
                case YAML_UTF8_ENCODING:

                    /*
                     * Decode a UTF-8 character.  Check RFC 3629
                     * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
                     *
                     * The following table (taken from the RFC) is used for
                     * decoding.
                     *
                     *    Char. number range |        UTF-8 octet sequence
                     *      (hexadecimal)    |              (binary)
                     *   --------------------+------------------------------------
                     *   0000 0000-0000 007F | 0xxxxxxx
                     *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
                     *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
                     *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                     *
                     * Additionally, the characters in the range 0xD800-0xDFFF
                     * are prohibited as they are reserved for use with UTF-16
                     * surrogate pairs.
                     */

                    /* Determine the length of the UTF-8 sequence. */

                    octet = parser->raw_pointer[0];
                    width = (octet & 0x80) == 0x00 ? 1 :
                            (octet & 0xE0) == 0xC0 ? 2 :
                            (octet & 0xF0) == 0xE0 ? 3 :
                            (octet & 0xF8) == 0xF0 ? 4 : 0;

                    /* Check if the leading octet is valid. */

                    if (!width)
                        return yaml_parser_set_reader_error(parser,
                                "Invalid leading UTF-8 octet",
                                parser->offset, octet);

                    /* Check if the raw buffer contains an incomplete character. */

                    if (width > parser->raw_unread) {
                        if (parser->eof) {
                            return yaml_parser_set_reader_error(parser,
                                    "Incomplete UTF-8 octet sequence",
                                    parser->offset, -1);
                        }
                        incomplete = 1;
                        break;
                    }

                    /* Decode the leading octet. */

                    value = (octet & 0x80) == 0x00 ? octet & 0x7F :
                            (octet & 0xE0) == 0xC0 ? octet & 0x1F :
                            (octet & 0xF0) == 0xE0 ? octet & 0x0F :
                            (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;

                    /* Check and decode the trailing octets. */

                    for (k = 1; k < width; k ++)
                    {
                        octet = parser->raw_pointer[k];

                        /* Check if the octet is valid. */

                        if ((octet & 0xC0) != 0x80)
                            return yaml_parser_set_reader_error(parser,
                                    "Invalid trailing UTF-8 octet",
                                    parser->offset+k, octet);

                        /* Decode the octet. */

                        value = (value << 6) + (octet & 0x3F);
                    }

                    /* Check the length of the sequence against the value. */

                    if (!((width == 1) ||
                            (width == 2 && value >= 0x80) ||
                            (width == 3 && value >= 0x800) ||
                            (width == 4 && value >= 0x10000)))
                        return yaml_parser_set_reader_error(parser,
                                "Invalid length of a UTF-8 sequence",
                                parser->offset, -1);

                    /* Check the range of the value. */

                    if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
                        return yaml_parser_set_reader_error(parser,
                                "Invalid Unicode character",
                                parser->offset, value);

                    break;
                
                case YAML_UTF16LE_ENCODING:
                case YAML_UTF16BE_ENCODING:

                    low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
                    high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);

                    /*
                     * The UTF-16 encoding is not as simple as one might
                     * naively think.  Check RFC 2781
                     * (http://www.ietf.org/rfc/rfc2781.txt).
                     *
                     * Normally, two subsequent bytes describe a Unicode
                     * character.  However a special technique (called a
                     * surrogate pair) is used for specifying character
                     * values larger than 0xFFFF.
                     *
                     * A surrogate pair consists of two pseudo-characters:
                     *      high surrogate area (0xD800-0xDBFF)
                     *      low surrogate area (0xDC00-0xDFFF)
                     *
                     * The following formulas are used for decoding
                     * and encoding characters using surrogate pairs:
                     * 
                     *  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
                     *  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
                     *  W1 = 110110yyyyyyyyyy
                     *  W2 = 110111xxxxxxxxxx
                     *
                     * where U is the character value, W1 is the high surrogate
                     * area, W2 is the low surrogate area.
                     */

                    /* Check for incomplete UTF-16 character. */

                    if (parser->raw_unread < 2) {
                        if (parser->eof) {
                            return yaml_parser_set_reader_error(parser,
                                    "Incomplete UTF-16 character",
                                    parser->offset, -1);
                        }
                        incomplete = 1;
                        break;
                    }

                    /* Get the character. */

                    value = parser->raw_pointer[low]
                        + (parser->raw_pointer[high] << 8);

                    /* Check for unexpected low surrogate area. */

                    if ((value & 0xFC00) == 0xDC00)
                        return yaml_parser_set_reader_error(parser,
                                "Unexpected low surrogate area",
                                parser->offset, value);

                    /* Check for a high surrogate area. */

                    if ((value & 0xFC00) == 0xD800) {

                        width = 4;

                        /* Check for incomplete surrogate pair. */

                        if (parser->raw_unread < 4) {
                            if (parser->eof) {
                                return yaml_parser_set_reader_error(parser,
                                        "Incomplete UTF-16 surrogate pair",
                                        parser->offset, -1);
                            }
                            incomplete = 1;
                            break;
                        }

                        /* Get the next character. */

                        unsigned int value2 = parser->raw_pointer[low+2]
                            + (parser->raw_pointer[high+2] << 8);

                        /* Check for a low surrogate area. */

                        if ((value2 & 0xFC00) != 0xDC00)
                            return yaml_parser_set_reader_error(parser,
                                    "Expected low surrogate area",
                                    parser->offset+2, value2);

                        /* Generate the value of the surrogate pair. */

                        value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
                    }

                    else {
                        width = 2;
                    }

                    break;
            }

            /* Check if the raw buffer contains enough bytes to form a character. */

            if (incomplete) break;

            /*
             * Check if the character is in the allowed range:
             *      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
             *      | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD]    (16 bit)
             *      | [#x10000-#x10FFFF]                        (32 bit)
             */

            if (! (value == 0x09 || value == 0x0A || value == 0x0D
                        || (value >= 0x20 && value <= 0x7E)
                        || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF)
                        || (value >= 0xE000 && value <= 0xFFFD)
                        || (value >= 0x10000 && value <= 0x10FFFF)))
                return yaml_parser_set_reader_error(parser,
                        "Control characters are not allowed",
                        parser->offset, value);

            /* Move the raw pointers. */

            parser->raw_pointer += width;
            parser->raw_unread -= width;
            parser->offset += width;

            /* Finally put the character into the buffer. */

            /* 0000 0000-0000 007F -> 0xxxxxxx */
            if (value <= 0x7F) {
                *(parser->buffer_end++) = value;
            }
            /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
            else if (value <= 0x7FF) {
                *(parser->buffer_end++) = 0xC0 + (value >> 6);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }
            /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
            else if (value <= 0xFFFF) {
                *(parser->buffer_end++) = 0xE0 + (value >> 12);
                *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }
            /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
            else {
                *(parser->buffer_end++) = 0xF0 + (value >> 18);
                *(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F);
                *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }

            parser->unread ++;
        }

        /* On EOF, put NUL into the buffer and return. */

        if (parser->eof) {
            *(parser->buffer_end++) = '\0';
            parser->unread ++;
            return 1;
        }

    }

    return 1;
}

/*
 * Determine the input stream encoding by checking the BOM symbol. If no BOM is
 * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
 */

#define BOM_UTF8    "\xef\xbb\xbf"
#define BOM_UTF16LE "\xff\xfe"
#define BOM_UTF16BE "\xfe\xff"

int
yaml_parser_determine_encoding(yaml_parser_t *parser)
{
    /* Ensure that we had enough bytes in the raw buffer. */

    while (!parser->eof && parser->raw_unread < 3) {
        if (!yaml_parser_update_raw_buffer(parser)) {
            return 0;
        }
    }

    /* Determine the encoding. */

    if (parser->raw_unread >= 2
            && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
        parser->encoding = YAML_UTF16LE_ENCODING;
        parser->raw_pointer += 2;
        parser->raw_unread -= 2;
        parser->offset += 2;
    }
    else if (parser->raw_unread >= 2
            && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
        parser->encoding = YAML_UTF16BE_ENCODING;
        parser->raw_pointer += 2;
        parser->raw_unread -= 2;
        parser->offset += 2;
    }
    else if (parser->raw_unread >= 3
            && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
        parser->encoding = YAML_UTF8_ENCODING;
        parser->raw_pointer += 3;
        parser->raw_unread -= 3;
        parser->offset += 3;
    }
    else {
        parser->encoding = YAML_UTF8_ENCODING;
    }

    return 1;
}

/*
 * Update the raw buffer.
 */

int
yaml_parser_update_raw_buffer(yaml_parser_t *parser)
{
    size_t size_read = 0;

    /* Return if the raw buffer is full. */

    if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;

    /* Return on EOF. */

    if (parser->eof) return 1;

    /* Move the remaining bytes in the raw buffer to the beginning. */

    if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
        memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
    }
    parser->raw_pointer = parser->raw_buffer;

    /* Call the read handler to fill the buffer. */

    if (!parser->read_handler(parser->read_handler_data,
                parser->raw_buffer + parser->raw_unread,
                YAML_RAW_BUFFER_SIZE - parser->raw_unread,
                &size_read)) {
        return yaml_parser_set_reader_error(parser, "Input error",
                parser->offset, -1);
    }
    parser->raw_unread += size_read;
    if (!size_read) {
        parser->eof = 1;
    }

    return 1;
}
Commit	Line	Data
95b98ba9	1
6eb1ded4 KS	2	#if HAVE_CONFIG_H
	3	#include <config.h>
	4	#endif
	5
	6	#include <yaml/yaml.h>
	7
	8	#include <assert.h>
	9
6eb1ded4	10	/*
3d790dfd	11	* Set the reader error and return 0.
6eb1ded4 KS	12	*/
6eb1ded4 KS	13
3d790dfd	14	int
2fa16060 KS	15	yaml_parser_set_reader_error(yaml_parser_t parser, const char problem,
2fa16060 KS	16	size_t offset, int value)
3d790dfd KS	17	{
	18	parser->error = YAML_READER_ERROR;
	19	parser->problem = problem;
2fa16060 KS	20	parser->problem_offset = offset;
2fa16060 KS	21	parser->problem_value = value;
6eb1ded4	22
3d790dfd KS	23	return 0;
3d790dfd KS	24	}
6eb1ded4	25
95b98ba9 KS	26
	27	/*
	28	* Ensure that the buffer contains at least length characters.
	29	* Return 1 on success, 0 on failure.
6eb1ded4 KS	30	*
6eb1ded4 KS	31	* The length is supposed to be significantly less that the buffer size.
95b98ba9 KS	32	*/
	33
	34	int
6eb1ded4	35	yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
95b98ba9	36	{
2fa16060	37	/* If the EOF flag is set and the raw buffer is empty, do nothing. */
95b98ba9	38
2fa16060	39	if (parser->eof && !parser->raw_unread)
95b98ba9 KS	40	return 1;
95b98ba9 KS	41
6eb1ded4 KS	42	/* Return if the buffer contains enough characters. */
	43
	44	if (parser->unread >= length)
	45	return 1;
	46
	47	/* Determine the input encoding if it is not known yet. */
95b98ba9	48
6eb1ded4 KS	49	if (!parser->encoding) {
6eb1ded4 KS	50	if (!yaml_parser_determine_encoding(parser))
95b98ba9	51	return 0;
95b98ba9 KS	52	}
95b98ba9 KS	53
6eb1ded4 KS	54	/* Move the unread characters to the beginning of the buffer. */
	55
	56	if (parser->buffer < parser->pointer
	57	&& parser->pointer < parser->buffer_end) {
	58	size_t size = parser->buffer_end - parser->pointer;
	59	memmove(parser->buffer, parser->pointer, size);
	60	parser->pointer = parser->buffer;
	61	parser->buffer_end -= size;
	62	}
	63	else if (parser->pointer == parser->buffer_end) {
	64	parser->pointer = parser->buffer;
	65	parser->buffer_end = parser->buffer;
	66	}
	67
	68	/* Fill the buffer until it has enough characters. */
	69
	70	while (parser->unread < length)
	71	{
	72	/* Fill the raw buffer. */
	73
	74	if (!yaml_parser_update_raw_buffer(parser)) return 0;
	75
	76	/* Decode the raw buffer. */
	77
	78	while (parser->raw_unread)
	79	{
3d790dfd	80	unsigned int value, value2;
6eb1ded4	81	int incomplete = 0;
2fa16060 KS	82	unsigned char octet;
2fa16060 KS	83	unsigned int width;
3d790dfd	84	int k, low, high;
6eb1ded4 KS	85
	86	/* Decode the next character. */
	87
	88	switch (parser->encoding)
	89	{
	90	case YAML_UTF8_ENCODING:
	91
3d790dfd KS	92	/*
	93	* Decode a UTF-8 character. Check RFC 3629
	94	* (http://www.ietf.org/rfc/rfc3629.txt) for more details.
	95	*
	96	* The following table (taken from the RFC) is used for
	97	* decoding.
	98	*
	99	* Char. number range \| UTF-8 octet sequence
	100	* (hexadecimal) \| (binary)
	101	* --------------------+------------------------------------
	102	* 0000 0000-0000 007F \| 0xxxxxxx
	103	* 0000 0080-0000 07FF \| 110xxxxx 10xxxxxx
	104	* 0000 0800-0000 FFFF \| 1110xxxx 10xxxxxx 10xxxxxx
	105	* 0001 0000-0010 FFFF \| 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
	106	*
	107	* Additionally, the characters in the range 0xD800-0xDFFF
	108	* are prohibited as they are reserved for use with UTF-16
	109	* surrogate pairs.
	110	*/
	111
	112	/* Determine the length of the UTF-8 sequence. */
	113
2fa16060 KS	114	octet = parser->raw_pointer[0];
	115	width = (octet & 0x80) == 0x00 ? 1 :
	116	(octet & 0xE0) == 0xC0 ? 2 :
	117	(octet & 0xF0) == 0xE0 ? 3 :
	118	(octet & 0xF8) == 0xF0 ? 4 : 0;
3d790dfd KS	119
	120	/* Check if the leading octet is valid. */
	121
2fa16060	122	if (!width)
3d790dfd	123	return yaml_parser_set_reader_error(parser,
2fa16060 KS	124	"Invalid leading UTF-8 octet",
2fa16060 KS	125	parser->offset, octet);
6eb1ded4 KS	126
	127	/* Check if the raw buffer contains an incomplete character. */
	128
2fa16060	129	if (width > parser->raw_unread) {
6eb1ded4	130	if (parser->eof) {
3d790dfd	131	return yaml_parser_set_reader_error(parser,
2fa16060 KS	132	"Incomplete UTF-8 octet sequence",
2fa16060 KS	133	parser->offset, -1);
6eb1ded4 KS	134	}
6eb1ded4 KS	135	incomplete = 1;
3d790dfd	136	break;
6eb1ded4 KS	137	}
6eb1ded4 KS	138
3d790dfd	139	/* Decode the leading octet. */
6eb1ded4	140
2fa16060 KS	141	value = (octet & 0x80) == 0x00 ? octet & 0x7F :
	142	(octet & 0xE0) == 0xC0 ? octet & 0x1F :
	143	(octet & 0xF0) == 0xE0 ? octet & 0x0F :
	144	(octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
3d790dfd KS	145
	146	/* Check and decode the trailing octets. */
	147
2fa16060	148	for (k = 1; k < width; k ++)
3d790dfd	149	{
2fa16060	150	octet = parser->raw_pointer[k];
3d790dfd KS	151
	152	/* Check if the octet is valid. */
	153
2fa16060	154	if ((octet & 0xC0) != 0x80)
3d790dfd	155	return yaml_parser_set_reader_error(parser,
2fa16060 KS	156	"Invalid trailing UTF-8 octet",
2fa16060 KS	157	parser->offset+k, octet);
3d790dfd KS	158
	159	/* Decode the octet. */
	160
2fa16060	161	value = (value << 6) + (octet & 0x3F);
6eb1ded4 KS	162	}
6eb1ded4 KS	163
3d790dfd KS	164	/* Check the length of the sequence against the value. */
3d790dfd KS	165
2fa16060 KS	166	if (!((width == 1) \|\|
	167	(width == 2 && value >= 0x80) \|\|
	168	(width == 3 && value >= 0x800) \|\|
	169	(width == 4 && value >= 0x10000)))
3d790dfd	170	return yaml_parser_set_reader_error(parser,
2fa16060 KS	171	"Invalid length of a UTF-8 sequence",
2fa16060 KS	172	parser->offset, -1);
3d790dfd KS	173
	174	/* Check the range of the value. */
	175
	176	if ((value >= 0xD800 && value <= 0xDFFF) \|\| value > 0x10FFFF)
	177	return yaml_parser_set_reader_error(parser,
2fa16060 KS	178	"Invalid Unicode character",
2fa16060 KS	179	parser->offset, value);
3d790dfd	180
6eb1ded4 KS	181	break;
	182
	183	case YAML_UTF16LE_ENCODING:
3d790dfd	184	case YAML_UTF16BE_ENCODING:
6eb1ded4	185
3d790dfd KS	186	low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
	187	high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);
	188
	189	/*
	190	* The UTF-16 encoding is not as simple as one might
	191	* naively think. Check RFC 2781
	192	* (http://www.ietf.org/rfc/rfc2781.txt).
	193	*
	194	* Normally, two subsequent bytes describe a Unicode
	195	* character. However a special technique (called a
	196	* surrogate pair) is used for specifying character
	197	* values larger than 0xFFFF.
	198	*
	199	* A surrogate pair consists of two pseudo-characters:
	200	* high surrogate area (0xD800-0xDBFF)
	201	* low surrogate area (0xDC00-0xDFFF)
	202	*
	203	* The following formulas are used for decoding
	204	* and encoding characters using surrogate pairs:
	205	*
	206	* U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
	207	* U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
	208	* W1 = 110110yyyyyyyyyy
	209	* W2 = 110111xxxxxxxxxx
	210	*
	211	* where U is the character value, W1 is the high surrogate
	212	* area, W2 is the low surrogate area.
	213	*/
	214
	215	/* Check for incomplete UTF-16 character. */
6eb1ded4 KS	216
	217	if (parser->raw_unread < 2) {
	218	if (parser->eof) {
3d790dfd	219	return yaml_parser_set_reader_error(parser,
2fa16060 KS	220	"Incomplete UTF-16 character",
2fa16060 KS	221	parser->offset, -1);
6eb1ded4 KS	222	}
6eb1ded4 KS	223	incomplete = 1;
3d790dfd	224	break;
6eb1ded4 KS	225	}
6eb1ded4 KS	226
3d790dfd	227	/* Get the character. */
6eb1ded4	228
3d790dfd KS	229	value = parser->raw_pointer[low]
3d790dfd KS	230	+ (parser->raw_pointer[high] << 8);
6eb1ded4	231
3d790dfd	232	/* Check for unexpected low surrogate area. */
6eb1ded4	233
3d790dfd KS	234	if ((value & 0xFC00) == 0xDC00)
3d790dfd KS	235	return yaml_parser_set_reader_error(parser,
2fa16060 KS	236	"Unexpected low surrogate area",
2fa16060 KS	237	parser->offset, value);
6eb1ded4	238
3d790dfd	239	/* Check for a high surrogate area. */
6eb1ded4	240
3d790dfd KS	241	if ((value & 0xFC00) == 0xD800) {
3d790dfd KS	242
2fa16060 KS	243	width = 4;
2fa16060 KS	244
3d790dfd KS	245	/* Check for incomplete surrogate pair. */
	246
	247	if (parser->raw_unread < 4) {
	248	if (parser->eof) {
	249	return yaml_parser_set_reader_error(parser,
2fa16060 KS	250	"Incomplete UTF-16 surrogate pair",
2fa16060 KS	251	parser->offset, -1);
3d790dfd KS	252	}
	253	incomplete = 1;
	254	break;
6eb1ded4	255	}
6eb1ded4	256
3d790dfd	257	/* Get the next character. */
6eb1ded4	258
3d790dfd KS	259	unsigned int value2 = parser->raw_pointer[low+2]
	260	+ (parser->raw_pointer[high+2] << 8);
	261
	262	/* Check for a low surrogate area. */
	263
	264	if ((value2 & 0xFC00) != 0xDC00)
	265	return yaml_parser_set_reader_error(parser,
2fa16060 KS	266	"Expected low surrogate area",
2fa16060 KS	267	parser->offset+2, value2);
3d790dfd KS	268
	269	/* Generate the value of the surrogate pair. */
	270
	271	value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
3d790dfd KS	272	}
	273
	274	else {
2fa16060	275	width = 2;
3d790dfd	276	}
6eb1ded4 KS	277
	278	break;
	279	}
	280
3d790dfd KS	281	/* Check if the raw buffer contains enough bytes to form a character. */
	282
	283	if (incomplete) break;
	284
6eb1ded4 KS	285	/*
	286	* Check if the character is in the allowed range:
	287	* #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
	288	* \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
	289	* \| [#x10000-#x10FFFF] (32 bit)
	290	*/
	291
3d790dfd KS	292	if (! (value == 0x09 \|\| value == 0x0A \|\| value == 0x0D
	293	\|\| (value >= 0x20 && value <= 0x7E)
	294	\|\| (value == 0x85) \|\| (value >= 0xA0 && value <= 0xD7FF)
	295	\|\| (value >= 0xE000 && value <= 0xFFFD)
	296	\|\| (value >= 0x10000 && value <= 0x10FFFF)))
	297	return yaml_parser_set_reader_error(parser,
2fa16060 KS	298	"Control characters are not allowed",
	299	parser->offset, value);
	300
	301	/* Move the raw pointers. */
	302
	303	parser->raw_pointer += width;
	304	parser->raw_unread -= width;
	305	parser->offset += width;
6eb1ded4 KS	306
	307	/* Finally put the character into the buffer. */
	308
	309	/* 0000 0000-0000 007F -> 0xxxxxxx */
3d790dfd KS	310	if (value <= 0x7F) {
3d790dfd KS	311	*(parser->buffer_end++) = value;
6eb1ded4 KS	312	}
6eb1ded4 KS	313	/* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
3d790dfd KS	314	else if (value <= 0x7FF) {
3d790dfd KS	315	*(parser->buffer_end++) = 0xC0 + (value >> 6);
2fa16060	316	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4 KS	317	}
6eb1ded4 KS	318	/* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
3d790dfd KS	319	else if (value <= 0xFFFF) {
3d790dfd KS	320	*(parser->buffer_end++) = 0xE0 + (value >> 12);
2fa16060 KS	321	*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
2fa16060 KS	322	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4 KS	323	}
	324	/* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
	325	else {
3d790dfd	326	*(parser->buffer_end++) = 0xF0 + (value >> 18);
2fa16060 KS	327	*(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F);
	328	*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
	329	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4	330	}
2fa16060 KS	331
	332	parser->unread ++;
	333	}
	334
	335	/* On EOF, put NUL into the buffer and return. */
	336
	337	if (parser->eof) {
	338	*(parser->buffer_end++) = '\0';
	339	parser->unread ++;
	340	return 1;
95b98ba9	341	}
2fa16060	342
95b98ba9 KS	343	}
95b98ba9 KS	344
3d790dfd	345	return 1;
6eb1ded4	346	}
95b98ba9	347
6eb1ded4 KS	348	/*
	349	* Determine the input stream encoding by checking the BOM symbol. If no BOM is
	350	* found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
	351	*/
	352
3d790dfd KS	353	#define BOM_UTF8 "\xef\xbb\xbf"
	354	#define BOM_UTF16LE "\xff\xfe"
	355	#define BOM_UTF16BE "\xfe\xff"
	356
6eb1ded4 KS	357	int
	358	yaml_parser_determine_encoding(yaml_parser_t *parser)
	359	{
	360	/* Ensure that we had enough bytes in the raw buffer. */
	361
3d790dfd	362	while (!parser->eof && parser->raw_unread < 3) {
6eb1ded4	363	if (!yaml_parser_update_raw_buffer(parser)) {
95b98ba9	364	return 0;
6eb1ded4	365	}
95b98ba9 KS	366	}
95b98ba9 KS	367
6eb1ded4	368	/* Determine the encoding. */
95b98ba9	369
3d790dfd KS	370	if (parser->raw_unread >= 2
	371	&& !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
	372	parser->encoding = YAML_UTF16LE_ENCODING;
	373	parser->raw_pointer += 2;
	374	parser->raw_unread -= 2;
2fa16060	375	parser->offset += 2;
3d790dfd KS	376	}
	377	else if (parser->raw_unread >= 2
	378	&& !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
6eb1ded4	379	parser->encoding = YAML_UTF16BE_ENCODING;
3d790dfd KS	380	parser->raw_pointer += 2;
3d790dfd KS	381	parser->raw_unread -= 2;
2fa16060	382	parser->offset += 2;
6eb1ded4	383	}
3d790dfd KS	384	else if (parser->raw_unread >= 3
	385	&& !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
	386	parser->encoding = YAML_UTF8_ENCODING;
	387	parser->raw_pointer += 3;
	388	parser->raw_unread -= 3;
2fa16060	389	parser->offset += 3;
6eb1ded4 KS	390	}
	391	else {
	392	parser->encoding = YAML_UTF8_ENCODING;
	393	}
3d790dfd KS	394
3d790dfd KS	395	return 1;
95b98ba9 KS	396	}
95b98ba9 KS	397
6eb1ded4 KS	398	/*
	399	* Update the raw buffer.
	400	*/
	401
	402	int
	403	yaml_parser_update_raw_buffer(yaml_parser_t *parser)
	404	{
	405	size_t size_read = 0;
	406
	407	/* Return if the raw buffer is full. */
	408
	409	if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;
95b98ba9	410
6eb1ded4 KS	411	/* Return on EOF. */
	412
	413	if (parser->eof) return 1;
	414
	415	/* Move the remaining bytes in the raw buffer to the beginning. */
	416
	417	if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
	418	memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
	419	}
	420	parser->raw_pointer = parser->raw_buffer;
	421
	422	/* Call the read handler to fill the buffer. */
	423
	424	if (!parser->read_handler(parser->read_handler_data,
	425	parser->raw_buffer + parser->raw_unread,
	426	YAML_RAW_BUFFER_SIZE - parser->raw_unread,
	427	&size_read)) {
2fa16060 KS	428	return yaml_parser_set_reader_error(parser, "Input error",
2fa16060 KS	429	parser->offset, -1);
6eb1ded4 KS	430	}
	431	parser->raw_unread += size_read;
	432	if (!size_read) {
	433	parser->eof = 1;
	434	}
	435
	436	return 1;
	437	}
95b98ba9	438