[libyaml.git] / src / reader.c


#if HAVE_CONFIG_H
#include <config.h>
#endif

#include <yaml.h>

#include <assert.h>

/*
 * Set the reader error and return 0.
 */

static int
yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
        size_t offset, int value)
{
    parser->error = YAML_READER_ERROR;
    parser->problem = problem;
    parser->problem_offset = offset;
    parser->problem_value = value;

    return 0;
}

/*
 * Update the raw buffer.
 */

static int
yaml_parser_update_raw_buffer(yaml_parser_t *parser)
{
    size_t size_read = 0;

    /* Return if the raw buffer is full. */

    if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;

    /* Return on EOF. */

    if (parser->eof) return 1;

    /* Move the remaining bytes in the raw buffer to the beginning. */

    if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
        memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
    }
    parser->raw_pointer = parser->raw_buffer;

    /* Call the read handler to fill the buffer. */

    if (!parser->read_handler(parser->read_handler_data,
                parser->raw_buffer + parser->raw_unread,
                YAML_RAW_BUFFER_SIZE - parser->raw_unread,
                &size_read)) {
        return yaml_parser_set_reader_error(parser, "Input error",
                parser->offset, -1);
    }
    parser->raw_unread += size_read;
    if (!size_read) {
        parser->eof = 1;
    }

    return 1;
}

/*
 * Determine the input stream encoding by checking the BOM symbol. If no BOM is
 * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
 */

#define BOM_UTF8    "\xef\xbb\xbf"
#define BOM_UTF16LE "\xff\xfe"
#define BOM_UTF16BE "\xfe\xff"

static int
yaml_parser_determine_encoding(yaml_parser_t *parser)
{
    /* Ensure that we had enough bytes in the raw buffer. */

    while (!parser->eof && parser->raw_unread < 3) {
        if (!yaml_parser_update_raw_buffer(parser)) {
            return 0;
        }
    }

    /* Determine the encoding. */

    if (parser->raw_unread >= 2
            && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
        parser->encoding = YAML_UTF16LE_ENCODING;
        parser->raw_pointer += 2;
        parser->raw_unread -= 2;
        parser->offset += 2;
    }
    else if (parser->raw_unread >= 2
            && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
        parser->encoding = YAML_UTF16BE_ENCODING;
        parser->raw_pointer += 2;
        parser->raw_unread -= 2;
        parser->offset += 2;
    }
    else if (parser->raw_unread >= 3
            && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
        parser->encoding = YAML_UTF8_ENCODING;
        parser->raw_pointer += 3;
        parser->raw_unread -= 3;
        parser->offset += 3;
    }
    else {
        parser->encoding = YAML_UTF8_ENCODING;
    }

    return 1;
}

/*
 * Ensure that the buffer contains at least length characters.
 * Return 1 on success, 0 on failure.
 *
 * The length is supposed to be significantly less that the buffer size.
 */

YAML_DECLARE(int)
yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
{
    /* If the EOF flag is set and the raw buffer is empty, do nothing. */

    if (parser->eof && !parser->raw_unread)
        return 1;

    /* Return if the buffer contains enough characters. */

    if (parser->unread >= length)
        return 1;

    /* Determine the input encoding if it is not known yet. */

    if (!parser->encoding) {
        if (!yaml_parser_determine_encoding(parser))
            return 0;
    }

    /* Move the unread characters to the beginning of the buffer. */

    if (parser->buffer < parser->pointer
            && parser->pointer < parser->buffer_end) {
        size_t size = parser->buffer_end - parser->pointer;
        memmove(parser->buffer, parser->pointer, size);
        parser->pointer = parser->buffer;
        parser->buffer_end = parser->buffer + size;
    }
    else if (parser->pointer == parser->buffer_end) {
        parser->pointer = parser->buffer;
        parser->buffer_end = parser->buffer;
    }

    /* Fill the buffer until it has enough characters. */

    while (parser->unread < length)
    {
        /* Fill the raw buffer. */

        if (!yaml_parser_update_raw_buffer(parser)) return 0;

        /* Decode the raw buffer. */

        while (parser->raw_unread)
        {
            unsigned int value, value2;
            int incomplete = 0;
            unsigned char octet;
            unsigned int width;
            int k, low, high;

            /* Decode the next character. */

            switch (parser->encoding)
            {
                case YAML_UTF8_ENCODING:

                    /*
                     * Decode a UTF-8 character.  Check RFC 3629
                     * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
                     *
                     * The following table (taken from the RFC) is used for
                     * decoding.
                     *
                     *    Char. number range |        UTF-8 octet sequence
                     *      (hexadecimal)    |              (binary)
                     *   --------------------+------------------------------------
                     *   0000 0000-0000 007F | 0xxxxxxx
                     *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
                     *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
                     *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                     *
                     * Additionally, the characters in the range 0xD800-0xDFFF
                     * are prohibited as they are reserved for use with UTF-16
                     * surrogate pairs.
                     */

                    /* Determine the length of the UTF-8 sequence. */

                    octet = parser->raw_pointer[0];
                    width = (octet & 0x80) == 0x00 ? 1 :
                            (octet & 0xE0) == 0xC0 ? 2 :
                            (octet & 0xF0) == 0xE0 ? 3 :
                            (octet & 0xF8) == 0xF0 ? 4 : 0;

                    /* Check if the leading octet is valid. */

                    if (!width)
                        return yaml_parser_set_reader_error(parser,
                                "Invalid leading UTF-8 octet",
                                parser->offset, octet);

                    /* Check if the raw buffer contains an incomplete character. */

                    if (width > parser->raw_unread) {
                        if (parser->eof) {
                            return yaml_parser_set_reader_error(parser,
                                    "Incomplete UTF-8 octet sequence",
                                    parser->offset, -1);
                        }
                        incomplete = 1;
                        break;
                    }

                    /* Decode the leading octet. */

                    value = (octet & 0x80) == 0x00 ? octet & 0x7F :
                            (octet & 0xE0) == 0xC0 ? octet & 0x1F :
                            (octet & 0xF0) == 0xE0 ? octet & 0x0F :
                            (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;

                    /* Check and decode the trailing octets. */

                    for (k = 1; k < width; k ++)
                    {
                        octet = parser->raw_pointer[k];

                        /* Check if the octet is valid. */

                        if ((octet & 0xC0) != 0x80)
                            return yaml_parser_set_reader_error(parser,
                                    "Invalid trailing UTF-8 octet",
                                    parser->offset+k, octet);

                        /* Decode the octet. */

                        value = (value << 6) + (octet & 0x3F);
                    }

                    /* Check the length of the sequence against the value. */

                    if (!((width == 1) ||
                            (width == 2 && value >= 0x80) ||
                            (width == 3 && value >= 0x800) ||
                            (width == 4 && value >= 0x10000)))
                        return yaml_parser_set_reader_error(parser,
                                "Invalid length of a UTF-8 sequence",
                                parser->offset, -1);

                    /* Check the range of the value. */

                    if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
                        return yaml_parser_set_reader_error(parser,
                                "Invalid Unicode character",
                                parser->offset, value);

                    break;
                
                case YAML_UTF16LE_ENCODING:
                case YAML_UTF16BE_ENCODING:

                    low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
                    high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);

                    /*
                     * The UTF-16 encoding is not as simple as one might
                     * naively think.  Check RFC 2781
                     * (http://www.ietf.org/rfc/rfc2781.txt).
                     *
                     * Normally, two subsequent bytes describe a Unicode
                     * character.  However a special technique (called a
                     * surrogate pair) is used for specifying character
                     * values larger than 0xFFFF.
                     *
                     * A surrogate pair consists of two pseudo-characters:
                     *      high surrogate area (0xD800-0xDBFF)
                     *      low surrogate area (0xDC00-0xDFFF)
                     *
                     * The following formulas are used for decoding
                     * and encoding characters using surrogate pairs:
                     * 
                     *  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
                     *  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
                     *  W1 = 110110yyyyyyyyyy
                     *  W2 = 110111xxxxxxxxxx
                     *
                     * where U is the character value, W1 is the high surrogate
                     * area, W2 is the low surrogate area.
                     */

                    /* Check for incomplete UTF-16 character. */

                    if (parser->raw_unread < 2) {
                        if (parser->eof) {
                            return yaml_parser_set_reader_error(parser,
                                    "Incomplete UTF-16 character",
                                    parser->offset, -1);
                        }
                        incomplete = 1;
                        break;
                    }

                    /* Get the character. */

                    value = parser->raw_pointer[low]
                        + (parser->raw_pointer[high] << 8);

                    /* Check for unexpected low surrogate area. */

                    if ((value & 0xFC00) == 0xDC00)
                        return yaml_parser_set_reader_error(parser,
                                "Unexpected low surrogate area",
                                parser->offset, value);

                    /* Check for a high surrogate area. */

                    if ((value & 0xFC00) == 0xD800) {

                        width = 4;

                        /* Check for incomplete surrogate pair. */

                        if (parser->raw_unread < 4) {
                            if (parser->eof) {
                                return yaml_parser_set_reader_error(parser,
                                        "Incomplete UTF-16 surrogate pair",
                                        parser->offset, -1);
                            }
                            incomplete = 1;
                            break;
                        }

                        /* Get the next character. */

                        unsigned int value2 = parser->raw_pointer[low+2]
                            + (parser->raw_pointer[high+2] << 8);

                        /* Check for a low surrogate area. */

                        if ((value2 & 0xFC00) != 0xDC00)
                            return yaml_parser_set_reader_error(parser,
                                    "Expected low surrogate area",
                                    parser->offset+2, value2);

                        /* Generate the value of the surrogate pair. */

                        value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
                    }

                    else {
                        width = 2;
                    }

                    break;
            }

            /* Check if the raw buffer contains enough bytes to form a character. */

            if (incomplete) break;

            /*
             * Check if the character is in the allowed range:
             *      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
             *      | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD]    (16 bit)
             *      | [#x10000-#x10FFFF]                        (32 bit)
             */

            if (! (value == 0x09 || value == 0x0A || value == 0x0D
                        || (value >= 0x20 && value <= 0x7E)
                        || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF)
                        || (value >= 0xE000 && value <= 0xFFFD)
                        || (value >= 0x10000 && value <= 0x10FFFF)))
                return yaml_parser_set_reader_error(parser,
                        "Control characters are not allowed",
                        parser->offset, value);

            /* Move the raw pointers. */

            parser->raw_pointer += width;
            parser->raw_unread -= width;
            parser->offset += width;

            /* Finally put the character into the buffer. */

            /* 0000 0000-0000 007F -> 0xxxxxxx */
            if (value <= 0x7F) {
                *(parser->buffer_end++) = value;
            }
            /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
            else if (value <= 0x7FF) {
                *(parser->buffer_end++) = 0xC0 + (value >> 6);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }
            /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
            else if (value <= 0xFFFF) {
                *(parser->buffer_end++) = 0xE0 + (value >> 12);
                *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }
            /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
            else {
                *(parser->buffer_end++) = 0xF0 + (value >> 18);
                *(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F);
                *(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
                *(parser->buffer_end++) = 0x80 + (value & 0x3F);
            }

            parser->unread ++;
        }

        /* On EOF, put NUL into the buffer and return. */

        if (parser->eof) {
            *(parser->buffer_end++) = '\0';
            parser->unread ++;
            return 1;
        }

    }

    return 1;
}
Commit	Line	Data
95b98ba9	1
6eb1ded4 KS	2	#if HAVE_CONFIG_H
	3	#include <config.h>
	4	#endif
	5
5eff53a4	6	#include <yaml.h>
6eb1ded4 KS	7
	8	#include <assert.h>
	9
6eb1ded4	10	/*
3d790dfd	11	* Set the reader error and return 0.
6eb1ded4 KS	12	*/
6eb1ded4 KS	13
f642fd11	14	static int
2fa16060 KS	15	yaml_parser_set_reader_error(yaml_parser_t parser, const char problem,
2fa16060 KS	16	size_t offset, int value)
3d790dfd KS	17	{
	18	parser->error = YAML_READER_ERROR;
	19	parser->problem = problem;
2fa16060 KS	20	parser->problem_offset = offset;
2fa16060 KS	21	parser->problem_value = value;
6eb1ded4	22
3d790dfd KS	23	return 0;
3d790dfd KS	24	}
6eb1ded4	25
f642fd11 KS	26	/*
	27	* Update the raw buffer.
	28	*/
	29
	30	static int
	31	yaml_parser_update_raw_buffer(yaml_parser_t *parser)
	32	{
	33	size_t size_read = 0;
	34
	35	/* Return if the raw buffer is full. */
	36
	37	if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;
	38
	39	/* Return on EOF. */
	40
	41	if (parser->eof) return 1;
	42
	43	/* Move the remaining bytes in the raw buffer to the beginning. */
	44
	45	if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
	46	memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
	47	}
	48	parser->raw_pointer = parser->raw_buffer;
	49
	50	/* Call the read handler to fill the buffer. */
	51
	52	if (!parser->read_handler(parser->read_handler_data,
	53	parser->raw_buffer + parser->raw_unread,
	54	YAML_RAW_BUFFER_SIZE - parser->raw_unread,
	55	&size_read)) {
	56	return yaml_parser_set_reader_error(parser, "Input error",
	57	parser->offset, -1);
	58	}
	59	parser->raw_unread += size_read;
	60	if (!size_read) {
	61	parser->eof = 1;
	62	}
	63
	64	return 1;
	65	}
	66
	67	/*
	68	* Determine the input stream encoding by checking the BOM symbol. If no BOM is
	69	* found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
	70	*/
	71
	72	#define BOM_UTF8 "\xef\xbb\xbf"
	73	#define BOM_UTF16LE "\xff\xfe"
	74	#define BOM_UTF16BE "\xfe\xff"
	75
	76	static int
	77	yaml_parser_determine_encoding(yaml_parser_t *parser)
	78	{
	79	/* Ensure that we had enough bytes in the raw buffer. */
	80
	81	while (!parser->eof && parser->raw_unread < 3) {
	82	if (!yaml_parser_update_raw_buffer(parser)) {
	83	return 0;
	84	}
	85	}
	86
	87	/* Determine the encoding. */
	88
	89	if (parser->raw_unread >= 2
90	&& !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
91	parser->encoding = YAML_UTF16LE_ENCODING;
92	parser->raw_pointer += 2;
93	parser->raw_unread -= 2;
94	parser->offset += 2;
95	}
96	else if (parser->raw_unread >= 2
97	&& !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
98	parser->encoding = YAML_UTF16BE_ENCODING;
99	parser->raw_pointer += 2;
100	parser->raw_unread -= 2;
101	parser->offset += 2;
102	}
103	else if (parser->raw_unread >= 3
104	&& !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
105	parser->encoding = YAML_UTF8_ENCODING;
106	parser->raw_pointer += 3;
107	parser->raw_unread -= 3;
108	parser->offset += 3;
109	}
110	else {
111	parser->encoding = YAML_UTF8_ENCODING;
112	}
113
114	return 1;
115	}
95b98ba9 KS	116
	117	/*
	118	* Ensure that the buffer contains at least length characters.
	119	* Return 1 on success, 0 on failure.
6eb1ded4 KS	120	*
6eb1ded4 KS	121	* The length is supposed to be significantly less that the buffer size.
95b98ba9 KS	122	*/
95b98ba9 KS	123
f642fd11	124	YAML_DECLARE(int)
6eb1ded4	125	yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
95b98ba9	126	{
2fa16060	127	/* If the EOF flag is set and the raw buffer is empty, do nothing. */
95b98ba9	128
2fa16060	129	if (parser->eof && !parser->raw_unread)
95b98ba9 KS	130	return 1;
95b98ba9 KS	131
6eb1ded4 KS	132	/* Return if the buffer contains enough characters. */
	133
	134	if (parser->unread >= length)
	135	return 1;
	136
	137	/* Determine the input encoding if it is not known yet. */
95b98ba9	138
6eb1ded4 KS	139	if (!parser->encoding) {
6eb1ded4 KS	140	if (!yaml_parser_determine_encoding(parser))
95b98ba9	141	return 0;
95b98ba9 KS	142	}
95b98ba9 KS	143
6eb1ded4 KS	144	/* Move the unread characters to the beginning of the buffer. */
	145
	146	if (parser->buffer < parser->pointer
	147	&& parser->pointer < parser->buffer_end) {
	148	size_t size = parser->buffer_end - parser->pointer;
	149	memmove(parser->buffer, parser->pointer, size);
	150	parser->pointer = parser->buffer;
26687d7d	151	parser->buffer_end = parser->buffer + size;
6eb1ded4 KS	152	}
	153	else if (parser->pointer == parser->buffer_end) {
	154	parser->pointer = parser->buffer;
	155	parser->buffer_end = parser->buffer;
	156	}
	157
	158	/* Fill the buffer until it has enough characters. */
	159
	160	while (parser->unread < length)
	161	{
	162	/* Fill the raw buffer. */
	163
	164	if (!yaml_parser_update_raw_buffer(parser)) return 0;
	165
	166	/* Decode the raw buffer. */
	167
	168	while (parser->raw_unread)
	169	{
3d790dfd	170	unsigned int value, value2;
6eb1ded4	171	int incomplete = 0;
2fa16060 KS	172	unsigned char octet;
2fa16060 KS	173	unsigned int width;
3d790dfd	174	int k, low, high;
6eb1ded4 KS	175
	176	/* Decode the next character. */
	177
	178	switch (parser->encoding)
	179	{
	180	case YAML_UTF8_ENCODING:
	181
3d790dfd KS	182	/*
	183	* Decode a UTF-8 character. Check RFC 3629
	184	* (http://www.ietf.org/rfc/rfc3629.txt) for more details.
	185	*
	186	* The following table (taken from the RFC) is used for
	187	* decoding.
	188	*
	189	* Char. number range \| UTF-8 octet sequence
	190	* (hexadecimal) \| (binary)
	191	* --------------------+------------------------------------
	192	* 0000 0000-0000 007F \| 0xxxxxxx
	193	* 0000 0080-0000 07FF \| 110xxxxx 10xxxxxx
	194	* 0000 0800-0000 FFFF \| 1110xxxx 10xxxxxx 10xxxxxx
	195	* 0001 0000-0010 FFFF \| 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
	196	*
	197	* Additionally, the characters in the range 0xD800-0xDFFF
	198	* are prohibited as they are reserved for use with UTF-16
	199	* surrogate pairs.
	200	*/
	201
	202	/* Determine the length of the UTF-8 sequence. */
	203
2fa16060 KS	204	octet = parser->raw_pointer[0];
	205	width = (octet & 0x80) == 0x00 ? 1 :
	206	(octet & 0xE0) == 0xC0 ? 2 :
	207	(octet & 0xF0) == 0xE0 ? 3 :
	208	(octet & 0xF8) == 0xF0 ? 4 : 0;
3d790dfd KS	209
	210	/* Check if the leading octet is valid. */
	211
2fa16060	212	if (!width)
3d790dfd	213	return yaml_parser_set_reader_error(parser,
2fa16060 KS	214	"Invalid leading UTF-8 octet",
2fa16060 KS	215	parser->offset, octet);
6eb1ded4 KS	216
	217	/* Check if the raw buffer contains an incomplete character. */
	218
2fa16060	219	if (width > parser->raw_unread) {
6eb1ded4	220	if (parser->eof) {
3d790dfd	221	return yaml_parser_set_reader_error(parser,
2fa16060 KS	222	"Incomplete UTF-8 octet sequence",
2fa16060 KS	223	parser->offset, -1);
6eb1ded4 KS	224	}
6eb1ded4 KS	225	incomplete = 1;
3d790dfd	226	break;
6eb1ded4 KS	227	}
6eb1ded4 KS	228
3d790dfd	229	/* Decode the leading octet. */
6eb1ded4	230
2fa16060 KS	231	value = (octet & 0x80) == 0x00 ? octet & 0x7F :
	232	(octet & 0xE0) == 0xC0 ? octet & 0x1F :
	233	(octet & 0xF0) == 0xE0 ? octet & 0x0F :
	234	(octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
3d790dfd KS	235
	236	/* Check and decode the trailing octets. */
	237
2fa16060	238	for (k = 1; k < width; k ++)
3d790dfd	239	{
2fa16060	240	octet = parser->raw_pointer[k];
3d790dfd KS	241
	242	/* Check if the octet is valid. */
	243
2fa16060	244	if ((octet & 0xC0) != 0x80)
3d790dfd	245	return yaml_parser_set_reader_error(parser,
2fa16060 KS	246	"Invalid trailing UTF-8 octet",
2fa16060 KS	247	parser->offset+k, octet);
3d790dfd KS	248
	249	/* Decode the octet. */
	250
2fa16060	251	value = (value << 6) + (octet & 0x3F);
6eb1ded4 KS	252	}
6eb1ded4 KS	253
3d790dfd KS	254	/* Check the length of the sequence against the value. */
3d790dfd KS	255
2fa16060 KS	256	if (!((width == 1) \|\|
	257	(width == 2 && value >= 0x80) \|\|
	258	(width == 3 && value >= 0x800) \|\|
	259	(width == 4 && value >= 0x10000)))
3d790dfd	260	return yaml_parser_set_reader_error(parser,
2fa16060 KS	261	"Invalid length of a UTF-8 sequence",
2fa16060 KS	262	parser->offset, -1);
3d790dfd KS	263
	264	/* Check the range of the value. */
	265
	266	if ((value >= 0xD800 && value <= 0xDFFF) \|\| value > 0x10FFFF)
	267	return yaml_parser_set_reader_error(parser,
2fa16060 KS	268	"Invalid Unicode character",
2fa16060 KS	269	parser->offset, value);
3d790dfd	270
6eb1ded4 KS	271	break;
	272
	273	case YAML_UTF16LE_ENCODING:
3d790dfd	274	case YAML_UTF16BE_ENCODING:
6eb1ded4	275
3d790dfd KS	276	low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
	277	high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);
	278
	279	/*
	280	* The UTF-16 encoding is not as simple as one might
	281	* naively think. Check RFC 2781
	282	* (http://www.ietf.org/rfc/rfc2781.txt).
	283	*
	284	* Normally, two subsequent bytes describe a Unicode
	285	* character. However a special technique (called a
	286	* surrogate pair) is used for specifying character
	287	* values larger than 0xFFFF.
	288	*
	289	* A surrogate pair consists of two pseudo-characters:
	290	* high surrogate area (0xD800-0xDBFF)
	291	* low surrogate area (0xDC00-0xDFFF)
	292	*
	293	* The following formulas are used for decoding
	294	* and encoding characters using surrogate pairs:
	295	*
	296	* U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
	297	* U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
	298	* W1 = 110110yyyyyyyyyy
	299	* W2 = 110111xxxxxxxxxx
	300	*
	301	* where U is the character value, W1 is the high surrogate
	302	* area, W2 is the low surrogate area.
	303	*/
	304
	305	/* Check for incomplete UTF-16 character. */
6eb1ded4 KS	306
	307	if (parser->raw_unread < 2) {
	308	if (parser->eof) {
3d790dfd	309	return yaml_parser_set_reader_error(parser,
2fa16060 KS	310	"Incomplete UTF-16 character",
2fa16060 KS	311	parser->offset, -1);
6eb1ded4 KS	312	}
6eb1ded4 KS	313	incomplete = 1;
3d790dfd	314	break;
6eb1ded4 KS	315	}
6eb1ded4 KS	316
3d790dfd	317	/* Get the character. */
6eb1ded4	318
3d790dfd KS	319	value = parser->raw_pointer[low]
3d790dfd KS	320	+ (parser->raw_pointer[high] << 8);
6eb1ded4	321
3d790dfd	322	/* Check for unexpected low surrogate area. */
6eb1ded4	323
3d790dfd KS	324	if ((value & 0xFC00) == 0xDC00)
3d790dfd KS	325	return yaml_parser_set_reader_error(parser,
2fa16060 KS	326	"Unexpected low surrogate area",
2fa16060 KS	327	parser->offset, value);
6eb1ded4	328
3d790dfd	329	/* Check for a high surrogate area. */
6eb1ded4	330
3d790dfd KS	331	if ((value & 0xFC00) == 0xD800) {
3d790dfd KS	332
2fa16060 KS	333	width = 4;
2fa16060 KS	334
3d790dfd KS	335	/* Check for incomplete surrogate pair. */
	336
	337	if (parser->raw_unread < 4) {
	338	if (parser->eof) {
	339	return yaml_parser_set_reader_error(parser,
2fa16060 KS	340	"Incomplete UTF-16 surrogate pair",
2fa16060 KS	341	parser->offset, -1);
3d790dfd KS	342	}
	343	incomplete = 1;
	344	break;
6eb1ded4	345	}
6eb1ded4	346
3d790dfd	347	/* Get the next character. */
6eb1ded4	348
3d790dfd KS	349	unsigned int value2 = parser->raw_pointer[low+2]
	350	+ (parser->raw_pointer[high+2] << 8);
	351
	352	/* Check for a low surrogate area. */
	353
	354	if ((value2 & 0xFC00) != 0xDC00)
	355	return yaml_parser_set_reader_error(parser,
2fa16060 KS	356	"Expected low surrogate area",
2fa16060 KS	357	parser->offset+2, value2);
3d790dfd KS	358
	359	/* Generate the value of the surrogate pair. */
	360
	361	value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
3d790dfd KS	362	}
	363
	364	else {
2fa16060	365	width = 2;
3d790dfd	366	}
6eb1ded4 KS	367
	368	break;
	369	}
	370
3d790dfd KS	371	/* Check if the raw buffer contains enough bytes to form a character. */
	372
	373	if (incomplete) break;
	374
6eb1ded4 KS	375	/*
	376	* Check if the character is in the allowed range:
	377	* #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
	378	* \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
	379	* \| [#x10000-#x10FFFF] (32 bit)
	380	*/
	381
3d790dfd KS	382	if (! (value == 0x09 \|\| value == 0x0A \|\| value == 0x0D
	383	\|\| (value >= 0x20 && value <= 0x7E)
	384	\|\| (value == 0x85) \|\| (value >= 0xA0 && value <= 0xD7FF)
	385	\|\| (value >= 0xE000 && value <= 0xFFFD)
	386	\|\| (value >= 0x10000 && value <= 0x10FFFF)))
	387	return yaml_parser_set_reader_error(parser,
2fa16060 KS	388	"Control characters are not allowed",
	389	parser->offset, value);
	390
	391	/* Move the raw pointers. */
	392
	393	parser->raw_pointer += width;
	394	parser->raw_unread -= width;
	395	parser->offset += width;
6eb1ded4 KS	396
	397	/* Finally put the character into the buffer. */
	398
	399	/* 0000 0000-0000 007F -> 0xxxxxxx */
3d790dfd KS	400	if (value <= 0x7F) {
3d790dfd KS	401	*(parser->buffer_end++) = value;
6eb1ded4 KS	402	}
6eb1ded4 KS	403	/* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
3d790dfd KS	404	else if (value <= 0x7FF) {
3d790dfd KS	405	*(parser->buffer_end++) = 0xC0 + (value >> 6);
2fa16060	406	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4 KS	407	}
6eb1ded4 KS	408	/* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
3d790dfd KS	409	else if (value <= 0xFFFF) {
3d790dfd KS	410	*(parser->buffer_end++) = 0xE0 + (value >> 12);
2fa16060 KS	411	*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
2fa16060 KS	412	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4 KS	413	}
	414	/* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
	415	else {
3d790dfd	416	*(parser->buffer_end++) = 0xF0 + (value >> 18);
2fa16060 KS	417	*(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F);
	418	*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
	419	*(parser->buffer_end++) = 0x80 + (value & 0x3F);
6eb1ded4	420	}
2fa16060 KS	421
	422	parser->unread ++;
	423	}
	424
	425	/* On EOF, put NUL into the buffer and return. */
	426
	427	if (parser->eof) {
	428	*(parser->buffer_end++) = '\0';
	429	parser->unread ++;
	430	return 1;
95b98ba9	431	}
2fa16060	432
95b98ba9 KS	433	}
95b98ba9 KS	434
3d790dfd	435	return 1;
6eb1ded4	436	}
95b98ba9	437