blob: cdff6ae7bfb55fa463c261c4aa52f483fc49bfda [file] [log] [blame]
/*
* encoding.c : implements the encoding conversion functions needed for XML
*
* Related specs:
* rfc2044 (UTF-8 and UTF-16) F. Yergeau Alis Technologies
* rfc2781 UTF-16, an encoding of ISO 10646, P. Hoffman, F. Yergeau
* [ISO-10646] UTF-8 and UTF-16 in Annexes
* [ISO-8859-1] ISO Latin-1 characters codes.
* [UNICODE] The Unicode Consortium, "The Unicode Standard --
* Worldwide Character Encoding -- Version 1.0", Addison-
* Wesley, Volume 1, 1991, Volume 2, 1992. UTF-8 is
* described in Unicode Technical Report #4.
* [US-ASCII] Coded Character Set--7-bit American Standard Code for
* Information Interchange, ANSI X3.4-1986.
*
* See Copyright for the status of this software.
*
* daniel@veillard.com
*
* Original code for IsoLatin1 and UTF-16 by "Martin J. Duerst" <duerst@w3.org>
*/
#define IN_LIBXML
#include "libxml.h"
#include <string.h>
#include <limits.h>
#ifdef HAVE_CTYPE_H
#include <ctype.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef LIBXML_ICONV_ENABLED
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#endif
#include <libxml/encoding.h>
#include <libxml/xmlmemory.h>
#ifdef LIBXML_HTML_ENABLED
#include <libxml/HTMLparser.h>
#endif
#include <libxml/globals.h>
#include <libxml/xmlerror.h>
#include "buf.h"
#include "enc.h"
static xmlCharEncodingHandlerPtr xmlUTF16LEHandler = NULL;
static xmlCharEncodingHandlerPtr xmlUTF16BEHandler = NULL;
typedef struct _xmlCharEncodingAlias xmlCharEncodingAlias;
typedef xmlCharEncodingAlias *xmlCharEncodingAliasPtr;
struct _xmlCharEncodingAlias {
const char *name;
const char *alias;
};
static xmlCharEncodingAliasPtr xmlCharEncodingAliases = NULL;
static int xmlCharEncodingAliasesNb = 0;
static int xmlCharEncodingAliasesMax = 0;
#if defined(LIBXML_ICONV_ENABLED) || defined(LIBXML_ICU_ENABLED)
#if 0
#define DEBUG_ENCODING /* Define this to get encoding traces */
#endif
#else
#ifdef LIBXML_ISO8859X_ENABLED
static void xmlRegisterCharEncodingHandlersISO8859x (void);
#endif
#endif
static int xmlLittleEndian = 1;
/**
* xmlEncodingErrMemory:
* @extra: extra information
*
* Handle an out of memory condition
*/
static void
xmlEncodingErrMemory(const char *extra)
{
__xmlSimpleError(XML_FROM_I18N, XML_ERR_NO_MEMORY, NULL, NULL, extra);
}
/**
* xmlErrEncoding:
* @error: the error number
* @msg: the error message
*
* n encoding error
*/
static void LIBXML_ATTR_FORMAT(2,0)
xmlEncodingErr(xmlParserErrors error, const char *msg, const char *val)
{
__xmlRaiseError(NULL, NULL, NULL, NULL, NULL,
XML_FROM_I18N, error, XML_ERR_FATAL,
NULL, 0, val, NULL, NULL, 0, 0, msg, val);
}
#ifdef LIBXML_ICU_ENABLED
static uconv_t*
openIcuConverter(const char* name, int toUnicode)
{
UErrorCode status = U_ZERO_ERROR;
uconv_t *conv = (uconv_t *) xmlMalloc(sizeof(uconv_t));
if (conv == NULL)
return NULL;
conv->pivot_source = conv->pivot_buf;
conv->pivot_target = conv->pivot_buf;
conv->uconv = ucnv_open(name, &status);
if (U_FAILURE(status))
goto error;
status = U_ZERO_ERROR;
if (toUnicode) {
ucnv_setToUCallBack(conv->uconv, UCNV_TO_U_CALLBACK_STOP,
NULL, NULL, NULL, &status);
}
else {
ucnv_setFromUCallBack(conv->uconv, UCNV_FROM_U_CALLBACK_STOP,
NULL, NULL, NULL, &status);
}
if (U_FAILURE(status))
goto error;
status = U_ZERO_ERROR;
conv->utf8 = ucnv_open("UTF-8", &status);
if (U_SUCCESS(status))
return conv;
error:
if (conv->uconv)
ucnv_close(conv->uconv);
xmlFree(conv);
return NULL;
}
static void
closeIcuConverter(uconv_t *conv)
{
if (conv != NULL) {
ucnv_close(conv->uconv);
ucnv_close(conv->utf8);
xmlFree(conv);
}
}
#endif /* LIBXML_ICU_ENABLED */
/************************************************************************
* *
* Conversions To/From UTF8 encoding *
* *
************************************************************************/
/**
* asciiToUTF8:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of ASCII chars
* @inlen: the length of @in
*
* Take a block of ASCII chars in and try to convert it to an UTF-8
* block of chars out.
* Returns 0 if success, or -1 otherwise
* The value of @inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
asciiToUTF8(unsigned char* out, int *outlen,
const unsigned char* in, int *inlen) {
unsigned char* outstart = out;
const unsigned char* base = in;
const unsigned char* processed = in;
unsigned char* outend = out + *outlen;
const unsigned char* inend;
unsigned int c;
inend = in + (*inlen);
while ((in < inend) && (out - outstart + 5 < *outlen)) {
c= *in++;
if (out >= outend)
break;
if (c < 0x80) {
*out++ = c;
} else {
*outlen = out - outstart;
*inlen = processed - base;
return(-1);
}
processed = (const unsigned char*) in;
}
*outlen = out - outstart;
*inlen = processed - base;
return(*outlen);
}
#ifdef LIBXML_OUTPUT_ENABLED
/**
* UTF8Toascii:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
*
* Take a block of UTF-8 chars in and try to convert it to an ASCII
* block of chars out.
*
* Returns 0 if success, -2 if the transcoding fails, or -1 otherwise
* The value of @inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
UTF8Toascii(unsigned char* out, int *outlen,
const unsigned char* in, int *inlen) {
const unsigned char* processed = in;
const unsigned char* outend;
const unsigned char* outstart = out;
const unsigned char* instart = in;
const unsigned char* inend;
unsigned int c, d;
int trailing;
if ((out == NULL) || (outlen == NULL) || (inlen == NULL)) return(-1);
if (in == NULL) {
/*
* initialization nothing to do
*/
*outlen = 0;
*inlen = 0;
return(0);
}
inend = in + (*inlen);
outend = out + (*outlen);
while (in < inend) {
d = *in++;
if (d < 0x80) { c= d; trailing= 0; }
else if (d < 0xC0) {
/* trailing byte in leading position */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
} else if (d < 0xE0) { c= d & 0x1F; trailing= 1; }
else if (d < 0xF0) { c= d & 0x0F; trailing= 2; }
else if (d < 0xF8) { c= d & 0x07; trailing= 3; }
else {
/* no chance for this in Ascii */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
if (inend - in < trailing) {
break;
}
for ( ; trailing; trailing--) {
if ((in >= inend) || (((d= *in++) & 0xC0) != 0x80))
break;
c <<= 6;
c |= d & 0x3F;
}
/* assertion: c is a single UTF-4 value */
if (c < 0x80) {
if (out >= outend)
break;
*out++ = c;
} else {
/* no chance for this in Ascii */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
processed = in;
}
*outlen = out - outstart;
*inlen = processed - instart;
return(*outlen);
}
#endif /* LIBXML_OUTPUT_ENABLED */
/**
* isolat1ToUTF8:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of ISO Latin 1 chars
* @inlen: the length of @in
*
* Take a block of ISO Latin 1 chars in and try to convert it to an UTF-8
* block of chars out.
* Returns the number of bytes written if success, or -1 otherwise
* The value of @inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
int
isolat1ToUTF8(unsigned char* out, int *outlen,
const unsigned char* in, int *inlen) {
unsigned char* outstart = out;
const unsigned char* base = in;
unsigned char* outend;
const unsigned char* inend;
const unsigned char* instop;
if ((out == NULL) || (in == NULL) || (outlen == NULL) || (inlen == NULL))
return(-1);
outend = out + *outlen;
inend = in + (*inlen);
instop = inend;
while ((in < inend) && (out < outend - 1)) {
if (*in >= 0x80) {
*out++ = (((*in) >> 6) & 0x1F) | 0xC0;
*out++ = ((*in) & 0x3F) | 0x80;
++in;
}
if ((instop - in) > (outend - out)) instop = in + (outend - out);
while ((in < instop) && (*in < 0x80)) {
*out++ = *in++;
}
}
if ((in < inend) && (out < outend) && (*in < 0x80)) {
*out++ = *in++;
}
*outlen = out - outstart;
*inlen = in - base;
return(*outlen);
}
/**
* UTF8ToUTF8:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @inb: a pointer to an array of UTF-8 chars
* @inlenb: the length of @in in UTF-8 chars
*
* No op copy operation for UTF8 handling.
*
* Returns the number of bytes written, or -1 if lack of space.
* The value of *inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
*/
static int
UTF8ToUTF8(unsigned char* out, int *outlen,
const unsigned char* inb, int *inlenb)
{
int len;
if ((out == NULL) || (outlen == NULL) || (inlenb == NULL))
return(-1);
if (inb == NULL) {
/* inb == NULL means output is initialized. */
*outlen = 0;
*inlenb = 0;
return(0);
}
if (*outlen > *inlenb) {
len = *inlenb;
} else {
len = *outlen;
}
if (len < 0)
return(-1);
/*
* FIXME: Conversion functions must assure valid UTF-8, so we have
* to check for UTF-8 validity. Preferably, this converter shouldn't
* be used at all.
*/
memcpy(out, inb, len);
*outlen = len;
*inlenb = len;
return(*outlen);
}
#ifdef LIBXML_OUTPUT_ENABLED
/**
* UTF8Toisolat1:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
*
* Take a block of UTF-8 chars in and try to convert it to an ISO Latin 1
* block of chars out.
*
* Returns the number of bytes written if success, -2 if the transcoding fails,
or -1 otherwise
* The value of @inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
int
UTF8Toisolat1(unsigned char* out, int *outlen,
const unsigned char* in, int *inlen) {
const unsigned char* processed = in;
const unsigned char* outend;
const unsigned char* outstart = out;
const unsigned char* instart = in;
const unsigned char* inend;
unsigned int c, d;
int trailing;
if ((out == NULL) || (outlen == NULL) || (inlen == NULL)) return(-1);
if (in == NULL) {
/*
* initialization nothing to do
*/
*outlen = 0;
*inlen = 0;
return(0);
}
inend = in + (*inlen);
outend = out + (*outlen);
while (in < inend) {
d = *in++;
if (d < 0x80) { c= d; trailing= 0; }
else if (d < 0xC0) {
/* trailing byte in leading position */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
} else if (d < 0xE0) { c= d & 0x1F; trailing= 1; }
else if (d < 0xF0) { c= d & 0x0F; trailing= 2; }
else if (d < 0xF8) { c= d & 0x07; trailing= 3; }
else {
/* no chance for this in IsoLat1 */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
if (inend - in < trailing) {
break;
}
for ( ; trailing; trailing--) {
if (in >= inend)
break;
if (((d= *in++) & 0xC0) != 0x80) {
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
c <<= 6;
c |= d & 0x3F;
}
/* assertion: c is a single UTF-4 value */
if (c <= 0xFF) {
if (out >= outend)
break;
*out++ = c;
} else {
/* no chance for this in IsoLat1 */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
processed = in;
}
*outlen = out - outstart;
*inlen = processed - instart;
return(*outlen);
}
#endif /* LIBXML_OUTPUT_ENABLED */
/**
* UTF16LEToUTF8:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @inb: a pointer to an array of UTF-16LE passwd as a byte array
* @inlenb: the length of @in in UTF-16LE chars
*
* Take a block of UTF-16LE ushorts in and try to convert it to an UTF-8
* block of chars out. This function assumes the endian property
* is the same between the native type of this machine and the
* inputed one.
*
* Returns the number of bytes written, or -1 if lack of space, or -2
* if the transcoding fails (if *in is not a valid utf16 string)
* The value of *inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
*/
static int
UTF16LEToUTF8(unsigned char* out, int *outlen,
const unsigned char* inb, int *inlenb)
{
unsigned char* outstart = out;
const unsigned char* processed = inb;
unsigned char* outend;
unsigned short* in = (unsigned short*) inb;
unsigned short* inend;
unsigned int c, d, inlen;
unsigned char *tmp;
int bits;
if (*outlen == 0) {
*inlenb = 0;
return(0);
}
outend = out + *outlen;
if ((*inlenb % 2) == 1)
(*inlenb)--;
inlen = *inlenb / 2;
inend = in + inlen;
while ((in < inend) && (out - outstart + 5 < *outlen)) {
if (xmlLittleEndian) {
c= *in++;
} else {
tmp = (unsigned char *) in;
c = *tmp++;
c = c | (((unsigned int)*tmp) << 8);
in++;
}
if ((c & 0xFC00) == 0xD800) { /* surrogates */
if (in >= inend) { /* (in > inend) shouldn't happens */
break;
}
if (xmlLittleEndian) {
d = *in++;
} else {
tmp = (unsigned char *) in;
d = *tmp++;
d = d | (((unsigned int)*tmp) << 8);
in++;
}
if ((d & 0xFC00) == 0xDC00) {
c &= 0x03FF;
c <<= 10;
c |= d & 0x03FF;
c += 0x10000;
}
else {
*outlen = out - outstart;
*inlenb = processed - inb;
return(-2);
}
}
/* assertion: c is a single UTF-4 value */
if (out >= outend)
break;
if (c < 0x80) { *out++= c; bits= -6; }
else if (c < 0x800) { *out++= ((c >> 6) & 0x1F) | 0xC0; bits= 0; }
else if (c < 0x10000) { *out++= ((c >> 12) & 0x0F) | 0xE0; bits= 6; }
else { *out++= ((c >> 18) & 0x07) | 0xF0; bits= 12; }
for ( ; bits >= 0; bits-= 6) {
if (out >= outend)
break;
*out++= ((c >> bits) & 0x3F) | 0x80;
}
processed = (const unsigned char*) in;
}
*outlen = out - outstart;
*inlenb = processed - inb;
return(*outlen);
}
#ifdef LIBXML_OUTPUT_ENABLED
/**
* UTF8ToUTF16LE:
* @outb: a pointer to an array of bytes to store the result
* @outlen: the length of @outb
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
*
* Take a block of UTF-8 chars in and try to convert it to an UTF-16LE
* block of chars out.
*
* Returns the number of bytes written, or -1 if lack of space, or -2
* if the transcoding failed.
*/
static int
UTF8ToUTF16LE(unsigned char* outb, int *outlen,
const unsigned char* in, int *inlen)
{
unsigned short* out = (unsigned short*) outb;
const unsigned char* processed = in;
const unsigned char *const instart = in;
unsigned short* outstart= out;
unsigned short* outend;
const unsigned char* inend;
unsigned int c, d;
int trailing;
unsigned char *tmp;
unsigned short tmp1, tmp2;
/* UTF16LE encoding has no BOM */
if ((out == NULL) || (outlen == NULL) || (inlen == NULL)) return(-1);
if (in == NULL) {
*outlen = 0;
*inlen = 0;
return(0);
}
inend= in + *inlen;
outend = out + (*outlen / 2);
while (in < inend) {
d= *in++;
if (d < 0x80) { c= d; trailing= 0; }
else if (d < 0xC0) {
/* trailing byte in leading position */
*outlen = (out - outstart) * 2;
*inlen = processed - instart;
return(-2);
} else if (d < 0xE0) { c= d & 0x1F; trailing= 1; }
else if (d < 0xF0) { c= d & 0x0F; trailing= 2; }
else if (d < 0xF8) { c= d & 0x07; trailing= 3; }
else {
/* no chance for this in UTF-16 */
*outlen = (out - outstart) * 2;
*inlen = processed - instart;
return(-2);
}
if (inend - in < trailing) {
break;
}
for ( ; trailing; trailing--) {
if ((in >= inend) || (((d= *in++) & 0xC0) != 0x80))
break;
c <<= 6;
c |= d & 0x3F;
}
/* assertion: c is a single UTF-4 value */
if (c < 0x10000) {
if (out >= outend)
break;
if (xmlLittleEndian) {
*out++ = c;
} else {
tmp = (unsigned char *) out;
*tmp = c ;
*(tmp + 1) = c >> 8 ;
out++;
}
}
else if (c < 0x110000) {
if (out+1 >= outend)
break;
c -= 0x10000;
if (xmlLittleEndian) {
*out++ = 0xD800 | (c >> 10);
*out++ = 0xDC00 | (c & 0x03FF);
} else {
tmp1 = 0xD800 | (c >> 10);
tmp = (unsigned char *) out;
*tmp = (unsigned char) tmp1;
*(tmp + 1) = tmp1 >> 8;
out++;
tmp2 = 0xDC00 | (c & 0x03FF);
tmp = (unsigned char *) out;
*tmp = (unsigned char) tmp2;
*(tmp + 1) = tmp2 >> 8;
out++;
}
}
else
break;
processed = in;
}
*outlen = (out - outstart) * 2;
*inlen = processed - instart;
return(*outlen);
}
/**
* UTF8ToUTF16:
* @outb: a pointer to an array of bytes to store the result
* @outlen: the length of @outb
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
*
* Take a block of UTF-8 chars in and try to convert it to an UTF-16
* block of chars out.
*
* Returns the number of bytes written, or -1 if lack of space, or -2
* if the transcoding failed.
*/
static int
UTF8ToUTF16(unsigned char* outb, int *outlen,
const unsigned char* in, int *inlen)
{
if (in == NULL) {
/*
* initialization, add the Byte Order Mark for UTF-16LE
*/
if (*outlen >= 2) {
outb[0] = 0xFF;
outb[1] = 0xFE;
*outlen = 2;
*inlen = 0;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"Added FFFE Byte Order Mark\n");
#endif
return(2);
}
*outlen = 0;
*inlen = 0;
return(0);
}
return (UTF8ToUTF16LE(outb, outlen, in, inlen));
}
#endif /* LIBXML_OUTPUT_ENABLED */
/**
* UTF16BEToUTF8:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @inb: a pointer to an array of UTF-16 passed as a byte array
* @inlenb: the length of @in in UTF-16 chars
*
* Take a block of UTF-16 ushorts in and try to convert it to an UTF-8
* block of chars out. This function assumes the endian property
* is the same between the native type of this machine and the
* inputed one.
*
* Returns the number of bytes written, or -1 if lack of space, or -2
* if the transcoding fails (if *in is not a valid utf16 string)
* The value of *inlen after return is the number of octets consumed
* if the return value is positive, else unpredictable.
*/
static int
UTF16BEToUTF8(unsigned char* out, int *outlen,
const unsigned char* inb, int *inlenb)
{
unsigned char* outstart = out;
const unsigned char* processed = inb;
unsigned char* outend = out + *outlen;
unsigned short* in = (unsigned short*) inb;
unsigned short* inend;
unsigned int c, d, inlen;
unsigned char *tmp;
int bits;
if ((*inlenb % 2) == 1)
(*inlenb)--;
inlen = *inlenb / 2;
inend= in + inlen;
while (in < inend) {
if (xmlLittleEndian) {
tmp = (unsigned char *) in;
c = *tmp++;
c = c << 8;
c = c | (unsigned int) *tmp;
in++;
} else {
c= *in++;
}
if ((c & 0xFC00) == 0xD800) { /* surrogates */
if (in >= inend) { /* (in > inend) shouldn't happens */
*outlen = out - outstart;
*inlenb = processed - inb;
return(-2);
}
if (xmlLittleEndian) {
tmp = (unsigned char *) in;
d = *tmp++;
d = d << 8;
d = d | (unsigned int) *tmp;
in++;
} else {
d= *in++;
}
if ((d & 0xFC00) == 0xDC00) {
c &= 0x03FF;
c <<= 10;
c |= d & 0x03FF;
c += 0x10000;
}
else {
*outlen = out - outstart;
*inlenb = processed - inb;
return(-2);
}
}
/* assertion: c is a single UTF-4 value */
if (out >= outend)
break;
if (c < 0x80) { *out++= c; bits= -6; }
else if (c < 0x800) { *out++= ((c >> 6) & 0x1F) | 0xC0; bits= 0; }
else if (c < 0x10000) { *out++= ((c >> 12) & 0x0F) | 0xE0; bits= 6; }
else { *out++= ((c >> 18) & 0x07) | 0xF0; bits= 12; }
for ( ; bits >= 0; bits-= 6) {
if (out >= outend)
break;
*out++= ((c >> bits) & 0x3F) | 0x80;
}
processed = (const unsigned char*) in;
}
*outlen = out - outstart;
*inlenb = processed - inb;
return(*outlen);
}
#ifdef LIBXML_OUTPUT_ENABLED
/**
* UTF8ToUTF16BE:
* @outb: a pointer to an array of bytes to store the result
* @outlen: the length of @outb
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
*
* Take a block of UTF-8 chars in and try to convert it to an UTF-16BE
* block of chars out.
*
* Returns the number of byte written, or -1 by lack of space, or -2
* if the transcoding failed.
*/
static int
UTF8ToUTF16BE(unsigned char* outb, int *outlen,
const unsigned char* in, int *inlen)
{
unsigned short* out = (unsigned short*) outb;
const unsigned char* processed = in;
const unsigned char *const instart = in;
unsigned short* outstart= out;
unsigned short* outend;
const unsigned char* inend;
unsigned int c, d;
int trailing;
unsigned char *tmp;
unsigned short tmp1, tmp2;
/* UTF-16BE has no BOM */
if ((outb == NULL) || (outlen == NULL) || (inlen == NULL)) return(-1);
if (in == NULL) {
*outlen = 0;
*inlen = 0;
return(0);
}
inend= in + *inlen;
outend = out + (*outlen / 2);
while (in < inend) {
d= *in++;
if (d < 0x80) { c= d; trailing= 0; }
else if (d < 0xC0) {
/* trailing byte in leading position */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
} else if (d < 0xE0) { c= d & 0x1F; trailing= 1; }
else if (d < 0xF0) { c= d & 0x0F; trailing= 2; }
else if (d < 0xF8) { c= d & 0x07; trailing= 3; }
else {
/* no chance for this in UTF-16 */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
}
if (inend - in < trailing) {
break;
}
for ( ; trailing; trailing--) {
if ((in >= inend) || (((d= *in++) & 0xC0) != 0x80)) break;
c <<= 6;
c |= d & 0x3F;
}
/* assertion: c is a single UTF-4 value */
if (c < 0x10000) {
if (out >= outend) break;
if (xmlLittleEndian) {
tmp = (unsigned char *) out;
*tmp = c >> 8;
*(tmp + 1) = c;
out++;
} else {
*out++ = c;
}
}
else if (c < 0x110000) {
if (out+1 >= outend) break;
c -= 0x10000;
if (xmlLittleEndian) {
tmp1 = 0xD800 | (c >> 10);
tmp = (unsigned char *) out;
*tmp = tmp1 >> 8;
*(tmp + 1) = (unsigned char) tmp1;
out++;
tmp2 = 0xDC00 | (c & 0x03FF);
tmp = (unsigned char *) out;
*tmp = tmp2 >> 8;
*(tmp + 1) = (unsigned char) tmp2;
out++;
} else {
*out++ = 0xD800 | (c >> 10);
*out++ = 0xDC00 | (c & 0x03FF);
}
}
else
break;
processed = in;
}
*outlen = (out - outstart) * 2;
*inlen = processed - instart;
return(*outlen);
}
#endif /* LIBXML_OUTPUT_ENABLED */
/************************************************************************
* *
* Generic encoding handling routines *
* *
************************************************************************/
/**
* xmlDetectCharEncoding:
* @in: a pointer to the first bytes of the XML entity, must be at least
* 2 bytes long (at least 4 if encoding is UTF4 variant).
* @len: pointer to the length of the buffer
*
* Guess the encoding of the entity using the first bytes of the entity content
* according to the non-normative appendix F of the XML-1.0 recommendation.
*
* Returns one of the XML_CHAR_ENCODING_... values.
*/
xmlCharEncoding
xmlDetectCharEncoding(const unsigned char* in, int len)
{
if (in == NULL)
return(XML_CHAR_ENCODING_NONE);
if (len >= 4) {
if ((in[0] == 0x00) && (in[1] == 0x00) &&
(in[2] == 0x00) && (in[3] == 0x3C))
return(XML_CHAR_ENCODING_UCS4BE);
if ((in[0] == 0x3C) && (in[1] == 0x00) &&
(in[2] == 0x00) && (in[3] == 0x00))
return(XML_CHAR_ENCODING_UCS4LE);
if ((in[0] == 0x00) && (in[1] == 0x00) &&
(in[2] == 0x3C) && (in[3] == 0x00))
return(XML_CHAR_ENCODING_UCS4_2143);
if ((in[0] == 0x00) && (in[1] == 0x3C) &&
(in[2] == 0x00) && (in[3] == 0x00))
return(XML_CHAR_ENCODING_UCS4_3412);
if ((in[0] == 0x4C) && (in[1] == 0x6F) &&
(in[2] == 0xA7) && (in[3] == 0x94))
return(XML_CHAR_ENCODING_EBCDIC);
if ((in[0] == 0x3C) && (in[1] == 0x3F) &&
(in[2] == 0x78) && (in[3] == 0x6D))
return(XML_CHAR_ENCODING_UTF8);
/*
* Although not part of the recommendation, we also
* attempt an "auto-recognition" of UTF-16LE and
* UTF-16BE encodings.
*/
if ((in[0] == 0x3C) && (in[1] == 0x00) &&
(in[2] == 0x3F) && (in[3] == 0x00))
return(XML_CHAR_ENCODING_UTF16LE);
if ((in[0] == 0x00) && (in[1] == 0x3C) &&
(in[2] == 0x00) && (in[3] == 0x3F))
return(XML_CHAR_ENCODING_UTF16BE);
}
if (len >= 3) {
/*
* Errata on XML-1.0 June 20 2001
* We now allow an UTF8 encoded BOM
*/
if ((in[0] == 0xEF) && (in[1] == 0xBB) &&
(in[2] == 0xBF))
return(XML_CHAR_ENCODING_UTF8);
}
/* For UTF-16 we can recognize by the BOM */
if (len >= 2) {
if ((in[0] == 0xFE) && (in[1] == 0xFF))
return(XML_CHAR_ENCODING_UTF16BE);
if ((in[0] == 0xFF) && (in[1] == 0xFE))
return(XML_CHAR_ENCODING_UTF16LE);
}
return(XML_CHAR_ENCODING_NONE);
}
/**
* xmlCleanupEncodingAliases:
*
* Unregisters all aliases
*/
void
xmlCleanupEncodingAliases(void) {
int i;
if (xmlCharEncodingAliases == NULL)
return;
for (i = 0;i < xmlCharEncodingAliasesNb;i++) {
if (xmlCharEncodingAliases[i].name != NULL)
xmlFree((char *) xmlCharEncodingAliases[i].name);
if (xmlCharEncodingAliases[i].alias != NULL)
xmlFree((char *) xmlCharEncodingAliases[i].alias);
}
xmlCharEncodingAliasesNb = 0;
xmlCharEncodingAliasesMax = 0;
xmlFree(xmlCharEncodingAliases);
xmlCharEncodingAliases = NULL;
}
/**
* xmlGetEncodingAlias:
* @alias: the alias name as parsed, in UTF-8 format (ASCII actually)
*
* Lookup an encoding name for the given alias.
*
* Returns NULL if not found, otherwise the original name
*/
const char *
xmlGetEncodingAlias(const char *alias) {
int i;
char upper[100];
if (alias == NULL)
return(NULL);
if (xmlCharEncodingAliases == NULL)
return(NULL);
for (i = 0;i < 99;i++) {
upper[i] = toupper(alias[i]);
if (upper[i] == 0) break;
}
upper[i] = 0;
/*
* Walk down the list looking for a definition of the alias
*/
for (i = 0;i < xmlCharEncodingAliasesNb;i++) {
if (!strcmp(xmlCharEncodingAliases[i].alias, upper)) {
return(xmlCharEncodingAliases[i].name);
}
}
return(NULL);
}
/**
* xmlAddEncodingAlias:
* @name: the encoding name as parsed, in UTF-8 format (ASCII actually)
* @alias: the alias name as parsed, in UTF-8 format (ASCII actually)
*
* Registers an alias @alias for an encoding named @name. Existing alias
* will be overwritten.
*
* Returns 0 in case of success, -1 in case of error
*/
int
xmlAddEncodingAlias(const char *name, const char *alias) {
int i;
char upper[100];
if ((name == NULL) || (alias == NULL))
return(-1);
for (i = 0;i < 99;i++) {
upper[i] = toupper(alias[i]);
if (upper[i] == 0) break;
}
upper[i] = 0;
if (xmlCharEncodingAliases == NULL) {
xmlCharEncodingAliasesNb = 0;
xmlCharEncodingAliasesMax = 20;
xmlCharEncodingAliases = (xmlCharEncodingAliasPtr)
xmlMalloc(xmlCharEncodingAliasesMax * sizeof(xmlCharEncodingAlias));
if (xmlCharEncodingAliases == NULL)
return(-1);
} else if (xmlCharEncodingAliasesNb >= xmlCharEncodingAliasesMax) {
xmlCharEncodingAliasesMax *= 2;
xmlCharEncodingAliases = (xmlCharEncodingAliasPtr)
xmlRealloc(xmlCharEncodingAliases,
xmlCharEncodingAliasesMax * sizeof(xmlCharEncodingAlias));
}
/*
* Walk down the list looking for a definition of the alias
*/
for (i = 0;i < xmlCharEncodingAliasesNb;i++) {
if (!strcmp(xmlCharEncodingAliases[i].alias, upper)) {
/*
* Replace the definition.
*/
xmlFree((char *) xmlCharEncodingAliases[i].name);
xmlCharEncodingAliases[i].name = xmlMemStrdup(name);
return(0);
}
}
/*
* Add the definition
*/
xmlCharEncodingAliases[xmlCharEncodingAliasesNb].name = xmlMemStrdup(name);
xmlCharEncodingAliases[xmlCharEncodingAliasesNb].alias = xmlMemStrdup(upper);
xmlCharEncodingAliasesNb++;
return(0);
}
/**
* xmlDelEncodingAlias:
* @alias: the alias name as parsed, in UTF-8 format (ASCII actually)
*
* Unregisters an encoding alias @alias
*
* Returns 0 in case of success, -1 in case of error
*/
int
xmlDelEncodingAlias(const char *alias) {
int i;
if (alias == NULL)
return(-1);
if (xmlCharEncodingAliases == NULL)
return(-1);
/*
* Walk down the list looking for a definition of the alias
*/
for (i = 0;i < xmlCharEncodingAliasesNb;i++) {
if (!strcmp(xmlCharEncodingAliases[i].alias, alias)) {
xmlFree((char *) xmlCharEncodingAliases[i].name);
xmlFree((char *) xmlCharEncodingAliases[i].alias);
xmlCharEncodingAliasesNb--;
memmove(&xmlCharEncodingAliases[i], &xmlCharEncodingAliases[i + 1],
sizeof(xmlCharEncodingAlias) * (xmlCharEncodingAliasesNb - i));
return(0);
}
}
return(-1);
}
/**
* xmlParseCharEncoding:
* @name: the encoding name as parsed, in UTF-8 format (ASCII actually)
*
* Compare the string to the encoding schemes already known. Note
* that the comparison is case insensitive accordingly to the section
* [XML] 4.3.3 Character Encoding in Entities.
*
* Returns one of the XML_CHAR_ENCODING_... values or XML_CHAR_ENCODING_NONE
* if not recognized.
*/
xmlCharEncoding
xmlParseCharEncoding(const char* name)
{
const char *alias;
char upper[500];
int i;
if (name == NULL)
return(XML_CHAR_ENCODING_NONE);
/*
* Do the alias resolution
*/
alias = xmlGetEncodingAlias(name);
if (alias != NULL)
name = alias;
for (i = 0;i < 499;i++) {
upper[i] = toupper(name[i]);
if (upper[i] == 0) break;
}
upper[i] = 0;
if (!strcmp(upper, "")) return(XML_CHAR_ENCODING_NONE);
if (!strcmp(upper, "UTF-8")) return(XML_CHAR_ENCODING_UTF8);
if (!strcmp(upper, "UTF8")) return(XML_CHAR_ENCODING_UTF8);
/*
* NOTE: if we were able to parse this, the endianness of UTF16 is
* already found and in use
*/
if (!strcmp(upper, "UTF-16")) return(XML_CHAR_ENCODING_UTF16LE);
if (!strcmp(upper, "UTF16")) return(XML_CHAR_ENCODING_UTF16LE);
if (!strcmp(upper, "ISO-10646-UCS-2")) return(XML_CHAR_ENCODING_UCS2);
if (!strcmp(upper, "UCS-2")) return(XML_CHAR_ENCODING_UCS2);
if (!strcmp(upper, "UCS2")) return(XML_CHAR_ENCODING_UCS2);
/*
* NOTE: if we were able to parse this, the endianness of UCS4 is
* already found and in use
*/
if (!strcmp(upper, "ISO-10646-UCS-4")) return(XML_CHAR_ENCODING_UCS4LE);
if (!strcmp(upper, "UCS-4")) return(XML_CHAR_ENCODING_UCS4LE);
if (!strcmp(upper, "UCS4")) return(XML_CHAR_ENCODING_UCS4LE);
if (!strcmp(upper, "ISO-8859-1")) return(XML_CHAR_ENCODING_8859_1);
if (!strcmp(upper, "ISO-LATIN-1")) return(XML_CHAR_ENCODING_8859_1);
if (!strcmp(upper, "ISO LATIN 1")) return(XML_CHAR_ENCODING_8859_1);
if (!strcmp(upper, "ISO-8859-2")) return(XML_CHAR_ENCODING_8859_2);
if (!strcmp(upper, "ISO-LATIN-2")) return(XML_CHAR_ENCODING_8859_2);
if (!strcmp(upper, "ISO LATIN 2")) return(XML_CHAR_ENCODING_8859_2);
if (!strcmp(upper, "ISO-8859-3")) return(XML_CHAR_ENCODING_8859_3);
if (!strcmp(upper, "ISO-8859-4")) return(XML_CHAR_ENCODING_8859_4);
if (!strcmp(upper, "ISO-8859-5")) return(XML_CHAR_ENCODING_8859_5);
if (!strcmp(upper, "ISO-8859-6")) return(XML_CHAR_ENCODING_8859_6);
if (!strcmp(upper, "ISO-8859-7")) return(XML_CHAR_ENCODING_8859_7);
if (!strcmp(upper, "ISO-8859-8")) return(XML_CHAR_ENCODING_8859_8);
if (!strcmp(upper, "ISO-8859-9")) return(XML_CHAR_ENCODING_8859_9);
if (!strcmp(upper, "ISO-2022-JP")) return(XML_CHAR_ENCODING_2022_JP);
if (!strcmp(upper, "SHIFT_JIS")) return(XML_CHAR_ENCODING_SHIFT_JIS);
if (!strcmp(upper, "EUC-JP")) return(XML_CHAR_ENCODING_EUC_JP);
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext, "Unknown encoding %s\n", name);
#endif
return(XML_CHAR_ENCODING_ERROR);
}
/**
* xmlGetCharEncodingName:
* @enc: the encoding
*
* The "canonical" name for XML encoding.
* C.f. http://www.w3.org/TR/REC-xml#charencoding
* Section 4.3.3 Character Encoding in Entities
*
* Returns the canonical name for the given encoding
*/
const char*
xmlGetCharEncodingName(xmlCharEncoding enc) {
switch (enc) {
case XML_CHAR_ENCODING_ERROR:
return(NULL);
case XML_CHAR_ENCODING_NONE:
return(NULL);
case XML_CHAR_ENCODING_UTF8:
return("UTF-8");
case XML_CHAR_ENCODING_UTF16LE:
return("UTF-16");
case XML_CHAR_ENCODING_UTF16BE:
return("UTF-16");
case XML_CHAR_ENCODING_EBCDIC:
return("EBCDIC");
case XML_CHAR_ENCODING_UCS4LE:
return("ISO-10646-UCS-4");
case XML_CHAR_ENCODING_UCS4BE:
return("ISO-10646-UCS-4");
case XML_CHAR_ENCODING_UCS4_2143:
return("ISO-10646-UCS-4");
case XML_CHAR_ENCODING_UCS4_3412:
return("ISO-10646-UCS-4");
case XML_CHAR_ENCODING_UCS2:
return("ISO-10646-UCS-2");
case XML_CHAR_ENCODING_8859_1:
return("ISO-8859-1");
case XML_CHAR_ENCODING_8859_2:
return("ISO-8859-2");
case XML_CHAR_ENCODING_8859_3:
return("ISO-8859-3");
case XML_CHAR_ENCODING_8859_4:
return("ISO-8859-4");
case XML_CHAR_ENCODING_8859_5:
return("ISO-8859-5");
case XML_CHAR_ENCODING_8859_6:
return("ISO-8859-6");
case XML_CHAR_ENCODING_8859_7:
return("ISO-8859-7");
case XML_CHAR_ENCODING_8859_8:
return("ISO-8859-8");
case XML_CHAR_ENCODING_8859_9:
return("ISO-8859-9");
case XML_CHAR_ENCODING_2022_JP:
return("ISO-2022-JP");
case XML_CHAR_ENCODING_SHIFT_JIS:
return("Shift-JIS");
case XML_CHAR_ENCODING_EUC_JP:
return("EUC-JP");
case XML_CHAR_ENCODING_ASCII:
return(NULL);
}
return(NULL);
}
/************************************************************************
* *
* Char encoding handlers *
* *
************************************************************************/
/* the size should be growable, but it's not a big deal ... */
#define MAX_ENCODING_HANDLERS 50
static xmlCharEncodingHandlerPtr *handlers = NULL;
static int nbCharEncodingHandler = 0;
/*
* The default is UTF-8 for XML, that's also the default used for the
* parser internals, so the default encoding handler is NULL
*/
static xmlCharEncodingHandlerPtr xmlDefaultCharEncodingHandler = NULL;
/**
* xmlNewCharEncodingHandler:
* @name: the encoding name, in UTF-8 format (ASCII actually)
* @input: the xmlCharEncodingInputFunc to read that encoding
* @output: the xmlCharEncodingOutputFunc to write that encoding
*
* Create and registers an xmlCharEncodingHandler.
*
* Returns the xmlCharEncodingHandlerPtr created (or NULL in case of error).
*/
xmlCharEncodingHandlerPtr
xmlNewCharEncodingHandler(const char *name,
xmlCharEncodingInputFunc input,
xmlCharEncodingOutputFunc output) {
xmlCharEncodingHandlerPtr handler;
const char *alias;
char upper[500];
int i;
char *up = NULL;
/*
* Do the alias resolution
*/
alias = xmlGetEncodingAlias(name);
if (alias != NULL)
name = alias;
/*
* Keep only the uppercase version of the encoding.
*/
if (name == NULL) {
xmlEncodingErr(XML_I18N_NO_NAME,
"xmlNewCharEncodingHandler : no name !\n", NULL);
return(NULL);
}
for (i = 0;i < 499;i++) {
upper[i] = toupper(name[i]);
if (upper[i] == 0) break;
}
upper[i] = 0;
up = xmlMemStrdup(upper);
if (up == NULL) {
xmlEncodingErrMemory("xmlNewCharEncodingHandler : out of memory !\n");
return(NULL);
}
/*
* allocate and fill-up an handler block.
*/
handler = (xmlCharEncodingHandlerPtr)
xmlMalloc(sizeof(xmlCharEncodingHandler));
if (handler == NULL) {
xmlFree(up);
xmlEncodingErrMemory("xmlNewCharEncodingHandler : out of memory !\n");
return(NULL);
}
memset(handler, 0, sizeof(xmlCharEncodingHandler));
handler->input = input;
handler->output = output;
handler->name = up;
#ifdef LIBXML_ICONV_ENABLED
handler->iconv_in = NULL;
handler->iconv_out = NULL;
#endif
#ifdef LIBXML_ICU_ENABLED
handler->uconv_in = NULL;
handler->uconv_out = NULL;
#endif
/*
* registers and returns the handler.
*/
xmlRegisterCharEncodingHandler(handler);
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"Registered encoding handler for %s\n", name);
#endif
return(handler);
}
/**
* xmlInitCharEncodingHandlers:
*
* Initialize the char encoding support, it registers the default
* encoding supported.
* NOTE: while public, this function usually doesn't need to be called
* in normal processing.
*/
void
xmlInitCharEncodingHandlers(void) {
unsigned short int tst = 0x1234;
unsigned char *ptr = (unsigned char *) &tst;
if (handlers != NULL) return;
handlers = (xmlCharEncodingHandlerPtr *)
xmlMalloc(MAX_ENCODING_HANDLERS * sizeof(xmlCharEncodingHandlerPtr));
if (*ptr == 0x12) xmlLittleEndian = 0;
else if (*ptr == 0x34) xmlLittleEndian = 1;
else {
xmlEncodingErr(XML_ERR_INTERNAL_ERROR,
"Odd problem at endianness detection\n", NULL);
}
if (handlers == NULL) {
xmlEncodingErrMemory("xmlInitCharEncodingHandlers : out of memory !\n");
return;
}
xmlNewCharEncodingHandler("UTF-8", UTF8ToUTF8, UTF8ToUTF8);
#ifdef LIBXML_OUTPUT_ENABLED
xmlUTF16LEHandler =
xmlNewCharEncodingHandler("UTF-16LE", UTF16LEToUTF8, UTF8ToUTF16LE);
xmlUTF16BEHandler =
xmlNewCharEncodingHandler("UTF-16BE", UTF16BEToUTF8, UTF8ToUTF16BE);
xmlNewCharEncodingHandler("UTF-16", UTF16LEToUTF8, UTF8ToUTF16);
xmlNewCharEncodingHandler("ISO-8859-1", isolat1ToUTF8, UTF8Toisolat1);
xmlNewCharEncodingHandler("ASCII", asciiToUTF8, UTF8Toascii);
xmlNewCharEncodingHandler("US-ASCII", asciiToUTF8, UTF8Toascii);
#ifdef LIBXML_HTML_ENABLED
xmlNewCharEncodingHandler("HTML", NULL, UTF8ToHtml);
#endif
#else
xmlUTF16LEHandler =
xmlNewCharEncodingHandler("UTF-16LE", UTF16LEToUTF8, NULL);
xmlUTF16BEHandler =
xmlNewCharEncodingHandler("UTF-16BE", UTF16BEToUTF8, NULL);
xmlNewCharEncodingHandler("UTF-16", UTF16LEToUTF8, NULL);
xmlNewCharEncodingHandler("ISO-8859-1", isolat1ToUTF8, NULL);
xmlNewCharEncodingHandler("ASCII", asciiToUTF8, NULL);
xmlNewCharEncodingHandler("US-ASCII", asciiToUTF8, NULL);
#endif /* LIBXML_OUTPUT_ENABLED */
#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED)
#ifdef LIBXML_ISO8859X_ENABLED
xmlRegisterCharEncodingHandlersISO8859x ();
#endif
#endif
}
/**
* xmlCleanupCharEncodingHandlers:
*
* Cleanup the memory allocated for the char encoding support, it
* unregisters all the encoding handlers and the aliases.
*/
void
xmlCleanupCharEncodingHandlers(void) {
xmlCleanupEncodingAliases();
if (handlers == NULL) return;
for (;nbCharEncodingHandler > 0;) {
nbCharEncodingHandler--;
if (handlers[nbCharEncodingHandler] != NULL) {
if (handlers[nbCharEncodingHandler]->name != NULL)
xmlFree(handlers[nbCharEncodingHandler]->name);
xmlFree(handlers[nbCharEncodingHandler]);
}
}
xmlFree(handlers);
handlers = NULL;
nbCharEncodingHandler = 0;
xmlDefaultCharEncodingHandler = NULL;
}
/**
* xmlRegisterCharEncodingHandler:
* @handler: the xmlCharEncodingHandlerPtr handler block
*
* Register the char encoding handler, surprising, isn't it ?
*/
void
xmlRegisterCharEncodingHandler(xmlCharEncodingHandlerPtr handler) {
if (handlers == NULL) xmlInitCharEncodingHandlers();
if ((handler == NULL) || (handlers == NULL)) {
xmlEncodingErr(XML_I18N_NO_HANDLER,
"xmlRegisterCharEncodingHandler: NULL handler !\n", NULL);
goto free_handler;
}
if (nbCharEncodingHandler >= MAX_ENCODING_HANDLERS) {
xmlEncodingErr(XML_I18N_EXCESS_HANDLER,
"xmlRegisterCharEncodingHandler: Too many handler registered, see %s\n",
"MAX_ENCODING_HANDLERS");
goto free_handler;
}
handlers[nbCharEncodingHandler++] = handler;
return;
free_handler:
if (handler != NULL) {
if (handler->name != NULL) {
xmlFree(handler->name);
}
xmlFree(handler);
}
}
/**
* xmlGetCharEncodingHandler:
* @enc: an xmlCharEncoding value.
*
* Search in the registered set the handler able to read/write that encoding.
*
* Returns the handler or NULL if not found
*/
xmlCharEncodingHandlerPtr
xmlGetCharEncodingHandler(xmlCharEncoding enc) {
xmlCharEncodingHandlerPtr handler;
if (handlers == NULL) xmlInitCharEncodingHandlers();
switch (enc) {
case XML_CHAR_ENCODING_ERROR:
return(NULL);
case XML_CHAR_ENCODING_NONE:
return(NULL);
case XML_CHAR_ENCODING_UTF8:
return(NULL);
case XML_CHAR_ENCODING_UTF16LE:
return(xmlUTF16LEHandler);
case XML_CHAR_ENCODING_UTF16BE:
return(xmlUTF16BEHandler);
case XML_CHAR_ENCODING_EBCDIC:
handler = xmlFindCharEncodingHandler("EBCDIC");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("ebcdic");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("EBCDIC-US");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("IBM-037");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_UCS4BE:
handler = xmlFindCharEncodingHandler("ISO-10646-UCS-4");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS-4");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS4");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_UCS4LE:
handler = xmlFindCharEncodingHandler("ISO-10646-UCS-4");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS-4");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS4");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_UCS4_2143:
break;
case XML_CHAR_ENCODING_UCS4_3412:
break;
case XML_CHAR_ENCODING_UCS2:
handler = xmlFindCharEncodingHandler("ISO-10646-UCS-2");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS-2");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("UCS2");
if (handler != NULL) return(handler);
break;
/*
* We used to keep ISO Latin encodings native in the
* generated data. This led to so many problems that
* this has been removed. One can still change this
* back by registering no-ops encoders for those
*/
case XML_CHAR_ENCODING_8859_1:
handler = xmlFindCharEncodingHandler("ISO-8859-1");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_2:
handler = xmlFindCharEncodingHandler("ISO-8859-2");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_3:
handler = xmlFindCharEncodingHandler("ISO-8859-3");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_4:
handler = xmlFindCharEncodingHandler("ISO-8859-4");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_5:
handler = xmlFindCharEncodingHandler("ISO-8859-5");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_6:
handler = xmlFindCharEncodingHandler("ISO-8859-6");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_7:
handler = xmlFindCharEncodingHandler("ISO-8859-7");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_8:
handler = xmlFindCharEncodingHandler("ISO-8859-8");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_8859_9:
handler = xmlFindCharEncodingHandler("ISO-8859-9");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_2022_JP:
handler = xmlFindCharEncodingHandler("ISO-2022-JP");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_SHIFT_JIS:
handler = xmlFindCharEncodingHandler("SHIFT-JIS");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("SHIFT_JIS");
if (handler != NULL) return(handler);
handler = xmlFindCharEncodingHandler("Shift_JIS");
if (handler != NULL) return(handler);
break;
case XML_CHAR_ENCODING_EUC_JP:
handler = xmlFindCharEncodingHandler("EUC-JP");
if (handler != NULL) return(handler);
break;
default:
break;
}
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"No handler found for encoding %d\n", enc);
#endif
return(NULL);
}
/**
* xmlFindCharEncodingHandler:
* @name: a string describing the char encoding.
*
* Search in the registered set the handler able to read/write that encoding.
*
* Returns the handler or NULL if not found
*/
xmlCharEncodingHandlerPtr
xmlFindCharEncodingHandler(const char *name) {
const char *nalias;
const char *norig;
xmlCharEncoding alias;
#ifdef LIBXML_ICONV_ENABLED
xmlCharEncodingHandlerPtr enc;
iconv_t icv_in, icv_out;
#endif /* LIBXML_ICONV_ENABLED */
#ifdef LIBXML_ICU_ENABLED
xmlCharEncodingHandlerPtr encu;
uconv_t *ucv_in, *ucv_out;
#endif /* LIBXML_ICU_ENABLED */
char upper[100];
int i;
if (handlers == NULL) xmlInitCharEncodingHandlers();
if (name == NULL) return(xmlDefaultCharEncodingHandler);
if (name[0] == 0) return(xmlDefaultCharEncodingHandler);
/*
* Do the alias resolution
*/
norig = name;
nalias = xmlGetEncodingAlias(name);
if (nalias != NULL)
name = nalias;
/*
* Check first for directly registered encoding names
*/
for (i = 0;i < 99;i++) {
upper[i] = toupper(name[i]);
if (upper[i] == 0) break;
}
upper[i] = 0;
if (handlers != NULL) {
for (i = 0;i < nbCharEncodingHandler; i++) {
if (!strcmp(upper, handlers[i]->name)) {
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"Found registered handler for encoding %s\n", name);
#endif
return(handlers[i]);
}
}
}
#ifdef LIBXML_ICONV_ENABLED
/* check whether iconv can handle this */
icv_in = iconv_open("UTF-8", name);
icv_out = iconv_open(name, "UTF-8");
if (icv_in == (iconv_t) -1) {
icv_in = iconv_open("UTF-8", upper);
}
if (icv_out == (iconv_t) -1) {
icv_out = iconv_open(upper, "UTF-8");
}
if ((icv_in != (iconv_t) -1) && (icv_out != (iconv_t) -1)) {
enc = (xmlCharEncodingHandlerPtr)
xmlMalloc(sizeof(xmlCharEncodingHandler));
if (enc == NULL) {
iconv_close(icv_in);
iconv_close(icv_out);
return(NULL);
}
memset(enc, 0, sizeof(xmlCharEncodingHandler));
enc->name = xmlMemStrdup(name);
enc->input = NULL;
enc->output = NULL;
enc->iconv_in = icv_in;
enc->iconv_out = icv_out;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"Found iconv handler for encoding %s\n", name);
#endif
return enc;
} else if ((icv_in != (iconv_t) -1) || icv_out != (iconv_t) -1) {
xmlEncodingErr(XML_ERR_INTERNAL_ERROR,
"iconv : problems with filters for '%s'\n", name);
}
#endif /* LIBXML_ICONV_ENABLED */
#ifdef LIBXML_ICU_ENABLED
/* check whether icu can handle this */
ucv_in = openIcuConverter(name, 1);
ucv_out = openIcuConverter(name, 0);
if (ucv_in != NULL && ucv_out != NULL) {
encu = (xmlCharEncodingHandlerPtr)
xmlMalloc(sizeof(xmlCharEncodingHandler));
if (encu == NULL) {
closeIcuConverter(ucv_in);
closeIcuConverter(ucv_out);
return(NULL);
}
memset(encu, 0, sizeof(xmlCharEncodingHandler));
encu->name = xmlMemStrdup(name);
encu->input = NULL;
encu->output = NULL;
encu->uconv_in = ucv_in;
encu->uconv_out = ucv_out;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"Found ICU converter handler for encoding %s\n", name);
#endif
return encu;
} else if (ucv_in != NULL || ucv_out != NULL) {
closeIcuConverter(ucv_in);
closeIcuConverter(ucv_out);
xmlEncodingErr(XML_ERR_INTERNAL_ERROR,
"ICU converter : problems with filters for '%s'\n", name);
}
#endif /* LIBXML_ICU_ENABLED */
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"No handler found for encoding %s\n", name);
#endif
/*
* Fallback using the canonical names
*/
alias = xmlParseCharEncoding(norig);
if (alias != XML_CHAR_ENCODING_ERROR) {
const char* canon;
canon = xmlGetCharEncodingName(alias);
if ((canon != NULL) && (strcmp(name, canon))) {
return(xmlFindCharEncodingHandler(canon));
}
}
/* If "none of the above", give up */
return(NULL);
}
/************************************************************************
* *
* ICONV based generic conversion functions *
* *
************************************************************************/
#ifdef LIBXML_ICONV_ENABLED
/**
* xmlIconvWrapper:
* @cd: iconv converter data structure
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of input bytes
* @inlen: the length of @in
*
* Returns 0 if success, or
* -1 by lack of space, or
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
* -3 if there the last byte can't form a single output char.
*
* The value of @inlen after return is the number of octets consumed
* as the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
xmlIconvWrapper(iconv_t cd, unsigned char *out, int *outlen,
const unsigned char *in, int *inlen) {
size_t icv_inlen, icv_outlen;
const char *icv_in = (const char *) in;
char *icv_out = (char *) out;
int ret;
if ((out == NULL) || (outlen == NULL) || (inlen == NULL) || (in == NULL)) {
if (outlen != NULL) *outlen = 0;
return(-1);
}
icv_inlen = *inlen;
icv_outlen = *outlen;
ret = iconv(cd, (ICONV_CONST char **) &icv_in, &icv_inlen, &icv_out, &icv_outlen);
*inlen -= icv_inlen;
*outlen -= icv_outlen;
if ((icv_inlen != 0) || (ret == -1)) {
#ifdef EILSEQ
if (errno == EILSEQ) {
return -2;
} else
#endif
#ifdef E2BIG
if (errno == E2BIG) {
return -1;
} else
#endif
#ifdef EINVAL
if (errno == EINVAL) {
return -3;
} else
#endif
{
return -3;
}
}
return 0;
}
#endif /* LIBXML_ICONV_ENABLED */
/************************************************************************
* *
* ICU based generic conversion functions *
* *
************************************************************************/
#ifdef LIBXML_ICU_ENABLED
/**
* xmlUconvWrapper:
* @cd: ICU uconverter data structure
* @toUnicode : non-zero if toUnicode. 0 otherwise.
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of input bytes
* @inlen: the length of @in
* @flush: if true, indicates end of input
*
* Returns 0 if success, or
* -1 by lack of space, or
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
* -3 if there the last byte can't form a single output char.
*
* The value of @inlen after return is the number of octets consumed
* as the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
xmlUconvWrapper(uconv_t *cd, int toUnicode, unsigned char *out, int *outlen,
const unsigned char *in, int *inlen, int flush) {
const char *ucv_in = (const char *) in;
char *ucv_out = (char *) out;
UErrorCode err = U_ZERO_ERROR;
if ((out == NULL) || (outlen == NULL) || (inlen == NULL) || (in == NULL)) {
if (outlen != NULL) *outlen = 0;
return(-1);
}
if (toUnicode) {
/* encoding => UTF-16 => UTF-8 */
ucnv_convertEx(cd->utf8, cd->uconv, &ucv_out, ucv_out + *outlen,
&ucv_in, ucv_in + *inlen, cd->pivot_buf,
&cd->pivot_source, &cd->pivot_target,
cd->pivot_buf + ICU_PIVOT_BUF_SIZE, 0, flush, &err);
} else {
/* UTF-8 => UTF-16 => encoding */
ucnv_convertEx(cd->uconv, cd->utf8, &ucv_out, ucv_out + *outlen,
&ucv_in, ucv_in + *inlen, cd->pivot_buf,
&cd->pivot_source, &cd->pivot_target,
cd->pivot_buf + ICU_PIVOT_BUF_SIZE, 0, flush, &err);
}
*inlen = ucv_in - (const char*) in;
*outlen = ucv_out - (char *) out;
if (U_SUCCESS(err)) {
/* reset pivot buf if this is the last call for input (flush==TRUE) */
if (flush)
cd->pivot_source = cd->pivot_target = cd->pivot_buf;
return 0;
}
if (err == U_BUFFER_OVERFLOW_ERROR)
return -1;
if (err == U_INVALID_CHAR_FOUND || err == U_ILLEGAL_CHAR_FOUND)
return -2;
return -3;
}
#endif /* LIBXML_ICU_ENABLED */
/************************************************************************
* *
* The real API used by libxml for on-the-fly conversion *
* *
************************************************************************/
/**
* xmlEncInputChunk:
* @handler: encoding handler
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of input bytes
* @inlen: the length of @in
* @flush: flush (ICU-related)
*
* Returns 0 if success, or
* -1 by lack of space, or
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
* -3 if there the last byte can't form a single output char.
*
* The value of @inlen after return is the number of octets consumed
* as the return value is 0, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
xmlEncInputChunk(xmlCharEncodingHandler *handler, unsigned char *out,
int *outlen, const unsigned char *in, int *inlen, int flush) {
int ret;
(void)flush;
if (handler->input != NULL) {
ret = handler->input(out, outlen, in, inlen);
if (ret > 0)
ret = 0;
}
#ifdef LIBXML_ICONV_ENABLED
else if (handler->iconv_in != NULL) {
ret = xmlIconvWrapper(handler->iconv_in, out, outlen, in, inlen);
}
#endif /* LIBXML_ICONV_ENABLED */
#ifdef LIBXML_ICU_ENABLED
else if (handler->uconv_in != NULL) {
ret = xmlUconvWrapper(handler->uconv_in, 1, out, outlen, in, inlen,
flush);
}
#endif /* LIBXML_ICU_ENABLED */
else {
*outlen = 0;
*inlen = 0;
ret = -2;
}
return(ret);
}
/**
* xmlEncOutputChunk:
* @handler: encoding handler
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of input bytes
* @inlen: the length of @in
*
* Returns 0 if success, or
* -1 by lack of space, or
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
* -3 if there the last byte can't form a single output char.
* -4 if no output function was found.
*
* The value of @inlen after return is the number of octets consumed
* as the return value is 0, else unpredictable.
* The value of @outlen after return is the number of octets produced.
*/
static int
xmlEncOutputChunk(xmlCharEncodingHandler *handler, unsigned char *out,
int *outlen, const unsigned char *in, int *inlen) {
int ret;
if (handler->output != NULL) {
ret = handler->output(out, outlen, in, inlen);
if (ret > 0)
ret = 0;
}
#ifdef LIBXML_ICONV_ENABLED
else if (handler->iconv_out != NULL) {
ret = xmlIconvWrapper(handler->iconv_out, out, outlen, in, inlen);
}
#endif /* LIBXML_ICONV_ENABLED */
#ifdef LIBXML_ICU_ENABLED
else if (handler->uconv_out != NULL) {
ret = xmlUconvWrapper(handler->uconv_out, 0, out, outlen, in, inlen,
1);
}
#endif /* LIBXML_ICU_ENABLED */
else {
*outlen = 0;
*inlen = 0;
ret = -4;
}
return(ret);
}
/**
* xmlCharEncFirstLineInt:
* @handler: char encoding transformation data structure
* @out: an xmlBuffer for the output.
* @in: an xmlBuffer for the input
* @len: number of bytes to convert for the first line, or -1
*
* Front-end for the encoding handler input function, but handle only
* the very first line, i.e. limit itself to 45 chars.
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncFirstLineInt(xmlCharEncodingHandler *handler, xmlBufferPtr out,
xmlBufferPtr in, int len) {
int ret;
int written;
int toconv;
if (handler == NULL) return(-1);
if (out == NULL) return(-1);
if (in == NULL) return(-1);
/* calculate space available */
written = out->size - out->use - 1; /* count '\0' */
toconv = in->use;
/*
* echo '<?xml version="1.0" encoding="UCS4"?>' | wc -c => 38
* 45 chars should be sufficient to reach the end of the encoding
* declaration without going too far inside the document content.
* on UTF-16 this means 90bytes, on UCS4 this means 180
* The actual value depending on guessed encoding is passed as @len
* if provided
*/
if (len >= 0) {
if (toconv > len)
toconv = len;
} else {
if (toconv > 180)
toconv = 180;
}
if (toconv * 2 >= written) {
xmlBufferGrow(out, toconv * 2);
written = out->size - out->use - 1;
}
ret = xmlEncInputChunk(handler, &out->content[out->use], &written,
in->content, &toconv, 0);
xmlBufferShrink(in, toconv);
out->use += written;
out->content[out->use] = 0;
if (ret == -1) ret = -3;
#ifdef DEBUG_ENCODING
switch (ret) {
case 0:
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input\n",
toconv, written);
break;
case -1:
xmlGenericError(xmlGenericErrorContext,"converted %d bytes to %d bytes of input, %d left\n",
toconv, written, in->use);
break;
case -2:
xmlGenericError(xmlGenericErrorContext,
"input conversion failed due to input error\n");
break;
case -3:
xmlGenericError(xmlGenericErrorContext,"converted %d bytes to %d bytes of input, %d left\n",
toconv, written, in->use);
break;
default:
xmlGenericError(xmlGenericErrorContext,"Unknown input conversion failed %d\n", ret);
}
#endif /* DEBUG_ENCODING */
/*
* Ignore when input buffer is not on a boundary
*/
if (ret == -3) ret = 0;
if (ret == -1) ret = 0;
return(written ? written : ret);
}
/**
* xmlCharEncFirstLine:
* @handler: char encoding transformation data structure
* @out: an xmlBuffer for the output.
* @in: an xmlBuffer for the input
*
* Front-end for the encoding handler input function, but handle only
* the very first line, i.e. limit itself to 45 chars.
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncFirstLine(xmlCharEncodingHandler *handler, xmlBufferPtr out,
xmlBufferPtr in) {
return(xmlCharEncFirstLineInt(handler, out, in, -1));
}
/**
* xmlCharEncFirstLineInput:
* @input: a parser input buffer
* @len: number of bytes to convert for the first line, or -1
*
* Front-end for the encoding handler input function, but handle only
* the very first line. Point is that this is based on autodetection
* of the encoding and once that first line is converted we may find
* out that a different decoder is needed to process the input.
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncFirstLineInput(xmlParserInputBufferPtr input, int len)
{
int ret;
size_t written;
size_t toconv;
int c_in;
int c_out;
xmlBufPtr in;
xmlBufPtr out;
if ((input == NULL) || (input->encoder == NULL) ||
(input->buffer == NULL) || (input->raw == NULL))
return (-1);
out = input->buffer;
in = input->raw;
toconv = xmlBufUse(in);
if (toconv == 0)
return (0);
written = xmlBufAvail(out) - 1; /* count '\0' */
/*
* echo '<?xml version="1.0" encoding="UCS4"?>' | wc -c => 38
* 45 chars should be sufficient to reach the end of the encoding
* declaration without going too far inside the document content.
* on UTF-16 this means 90bytes, on UCS4 this means 180
* The actual value depending on guessed encoding is passed as @len
* if provided
*/
if (len >= 0) {
if (toconv > (unsigned int) len)
toconv = len;
} else {
if (toconv > 180)
toconv = 180;
}
if (toconv * 2 >= written) {
xmlBufGrow(out, toconv * 2);
written = xmlBufAvail(out) - 1;
}
if (written > 360)
written = 360;
c_in = toconv;
c_out = written;
ret = xmlEncInputChunk(input->encoder, xmlBufEnd(out), &c_out,
xmlBufContent(in), &c_in, 0);
xmlBufShrink(in, c_in);
xmlBufAddLen(out, c_out);
if (ret == -1)
ret = -3;
switch (ret) {
case 0:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input\n",
c_in, c_out);
#endif
break;
case -1:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
c_in, c_out, (int)xmlBufUse(in));
#endif
break;
case -3:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
c_in, c_out, (int)xmlBufUse(in));
#endif
break;
case -2: {
char buf[50];
const xmlChar *content = xmlBufContent(in);
snprintf(&buf[0], 49, "0x%02X 0x%02X 0x%02X 0x%02X",
content[0], content[1],
content[2], content[3]);
buf[49] = 0;
xmlEncodingErr(XML_I18N_CONV_FAILED,
"input conversion failed due to input error, bytes %s\n",
buf);
}
}
/*
* Ignore when input buffer is not on a boundary
*/
if (ret == -3) ret = 0;
if (ret == -1) ret = 0;
return(c_out ? c_out : ret);
}
/**
* xmlCharEncInput:
* @input: a parser input buffer
* @flush: try to flush all the raw buffer
*
* Generic front-end for the encoding handler on parser input
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncInput(xmlParserInputBufferPtr input, int flush)
{
int ret;
size_t written;
size_t toconv;
int c_in;
int c_out;
xmlBufPtr in;
xmlBufPtr out;
if ((input == NULL) || (input->encoder == NULL) ||
(input->buffer == NULL) || (input->raw == NULL))
return (-1);
out = input->buffer;
in = input->raw;
toconv = xmlBufUse(in);
if (toconv == 0)
return (0);
if ((toconv > 64 * 1024) && (flush == 0))
toconv = 64 * 1024;
written = xmlBufAvail(out);
if (written > 0)
written--; /* count '\0' */
if (toconv * 2 >= written) {
xmlBufGrow(out, toconv * 2);
written = xmlBufAvail(out);
if (written > 0)
written--; /* count '\0' */
}
if ((written > 128 * 1024) && (flush == 0))
written = 128 * 1024;
c_in = toconv;
c_out = written;
ret = xmlEncInputChunk(input->encoder, xmlBufEnd(out), &c_out,
xmlBufContent(in), &c_in, flush);
xmlBufShrink(in, c_in);
xmlBufAddLen(out, c_out);
if (ret == -1)
ret = -3;
switch (ret) {
case 0:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input\n",
c_in, c_out);
#endif
break;
case -1:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
c_in, c_out, (int)xmlBufUse(in));
#endif
break;
case -3:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
c_in, c_out, (int)xmlBufUse(in));
#endif
break;
case -2: {
char buf[50];
const xmlChar *content = xmlBufContent(in);
snprintf(&buf[0], 49, "0x%02X 0x%02X 0x%02X 0x%02X",
content[0], content[1],
content[2], content[3]);
buf[49] = 0;
xmlEncodingErr(XML_I18N_CONV_FAILED,
"input conversion failed due to input error, bytes %s\n",
buf);
}
}
/*
* Ignore when input buffer is not on a boundary
*/
if (ret == -3)
ret = 0;
return (c_out? c_out : ret);
}
/**
* xmlCharEncInFunc:
* @handler: char encoding transformation data structure
* @out: an xmlBuffer for the output.
* @in: an xmlBuffer for the input
*
* Generic front-end for the encoding handler input function
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncInFunc(xmlCharEncodingHandler * handler, xmlBufferPtr out,
xmlBufferPtr in)
{
int ret;
int written;
int toconv;
if (handler == NULL)
return (-1);
if (out == NULL)
return (-1);
if (in == NULL)
return (-1);
toconv = in->use;
if (toconv == 0)
return (0);
written = out->size - out->use -1; /* count '\0' */
if (toconv * 2 >= written) {
xmlBufferGrow(out, out->size + toconv * 2);
written = out->size - out->use - 1;
}
ret = xmlEncInputChunk(handler, &out->content[out->use], &written,
in->content, &toconv, 1);
xmlBufferShrink(in, toconv);
out->use += written;
out->content[out->use] = 0;
if (ret == -1)
ret = -3;
switch (ret) {
case 0:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input\n",
toconv, written);
#endif
break;
case -1:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
toconv, written, in->use);
#endif
break;
case -3:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of input, %d left\n",
toconv, written, in->use);
#endif
break;
case -2: {
char buf[50];
snprintf(&buf[0], 49, "0x%02X 0x%02X 0x%02X 0x%02X",
in->content[0], in->content[1],
in->content[2], in->content[3]);
buf[49] = 0;
xmlEncodingErr(XML_I18N_CONV_FAILED,
"input conversion failed due to input error, bytes %s\n",
buf);
}
}
/*
* Ignore when input buffer is not on a boundary
*/
if (ret == -3)
ret = 0;
return (written? written : ret);
}
#ifdef LIBXML_OUTPUT_ENABLED
/**
* xmlCharEncOutput:
* @output: a parser output buffer
* @init: is this an initialization call without data
*
* Generic front-end for the encoding handler on parser output
* a first call with @init == 1 has to be made first to initiate the
* output in case of non-stateless encoding needing to initiate their
* state or the output (like the BOM in UTF16).
* In case of UTF8 sequence conversion errors for the given encoder,
* the content will be automatically remapped to a CharRef sequence.
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncOutput(xmlOutputBufferPtr output, int init)
{
int ret;
size_t written;
int writtentot = 0;
size_t toconv;
int c_in;
int c_out;
xmlBufPtr in;
xmlBufPtr out;
if ((output == NULL) || (output->encoder == NULL) ||
(output->buffer == NULL) || (output->conv == NULL))
return (-1);
out = output->conv;
in = output->buffer;
retry:
written = xmlBufAvail(out);
if (written > 0)
written--; /* count '\0' */
/*
* First specific handling of the initialization call
*/
if (init) {
c_in = 0;
c_out = written;
/* TODO: Check return value. */
xmlEncOutputChunk(output->encoder, xmlBufEnd(out), &c_out,
NULL, &c_in);
xmlBufAddLen(out, c_out);
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"initialized encoder\n");
#endif
return(c_out);
}
/*
* Conversion itself.
*/
toconv = xmlBufUse(in);
if (toconv == 0)
return (0);
if (toconv > 64 * 1024)
toconv = 64 * 1024;
if (toconv * 4 >= written) {
xmlBufGrow(out, toconv * 4);
written = xmlBufAvail(out) - 1;
}
if (written > 256 * 1024)
written = 256 * 1024;
c_in = toconv;
c_out = written;
ret = xmlEncOutputChunk(output->encoder, xmlBufEnd(out), &c_out,
xmlBufContent(in), &c_in);
xmlBufShrink(in, c_in);
xmlBufAddLen(out, c_out);
writtentot += c_out;
if (ret == -1) {
if (c_out > 0) {
/* Can be a limitation of iconv or uconv */
goto retry;
}
ret = -3;
}
/*
* Attempt to handle error cases
*/
switch (ret) {
case 0:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of output\n",
c_in, c_out);
#endif
break;
case -1:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"output conversion failed by lack of space\n");
#endif
break;
case -3:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,"converted %d bytes to %d bytes of output %d left\n",
c_in, c_out, (int) xmlBufUse(in));
#endif
break;
case -4:
xmlEncodingErr(XML_I18N_NO_OUTPUT,
"xmlCharEncOutFunc: no output function !\n", NULL);
ret = -1;
break;
case -2: {
xmlChar charref[20];
int len = (int) xmlBufUse(in);
xmlChar *content = xmlBufContent(in);
int cur, charrefLen;
cur = xmlGetUTF8Char(content, &len);
if (cur <= 0)
break;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"handling output conversion error\n");
xmlGenericError(xmlGenericErrorContext,
"Bytes: 0x%02X 0x%02X 0x%02X 0x%02X\n",
content[0], content[1],
content[2], content[3]);
#endif
/*
* Removes the UTF8 sequence, and replace it by a charref
* and continue the transcoding phase, hoping the error
* did not mangle the encoder state.
*/
charrefLen = snprintf((char *) &charref[0], sizeof(charref),
"&#%d;", cur);
xmlBufShrink(in, len);
xmlBufGrow(out, charrefLen * 4);
c_out = xmlBufAvail(out) - 1;
c_in = charrefLen;
ret = xmlEncOutputChunk(output->encoder, xmlBufEnd(out), &c_out,
charref, &c_in);
if ((ret < 0) || (c_in != charrefLen)) {
char buf[50];
snprintf(&buf[0], 49, "0x%02X 0x%02X 0x%02X 0x%02X",
content[0], content[1],
content[2], content[3]);
buf[49] = 0;
xmlEncodingErr(XML_I18N_CONV_FAILED,
"output conversion failed due to conv error, bytes %s\n",
buf);
if (xmlBufGetAllocationScheme(in) != XML_BUFFER_ALLOC_IMMUTABLE)
content[0] = ' ';
break;
}
xmlBufAddLen(out, c_out);
writtentot += c_out;
goto retry;
}
}
return(writtentot ? writtentot : ret);
}
#endif
/**
* xmlCharEncOutFunc:
* @handler: char encoding transformation data structure
* @out: an xmlBuffer for the output.
* @in: an xmlBuffer for the input
*
* Generic front-end for the encoding handler output function
* a first call with @in == NULL has to be made firs to initiate the
* output in case of non-stateless encoding needing to initiate their
* state or the output (like the BOM in UTF16).
* In case of UTF8 sequence conversion errors for the given encoder,
* the content will be automatically remapped to a CharRef sequence.
*
* Returns the number of byte written if success, or
* -1 general error
* -2 if the transcoding fails (for *in is not valid utf8 string or
* the result of transformation can't fit into the encoding we want), or
*/
int
xmlCharEncOutFunc(xmlCharEncodingHandler *handler, xmlBufferPtr out,
xmlBufferPtr in) {
int ret;
int written;
int writtentot = 0;
int toconv;
int output = 0;
if (handler == NULL) return(-1);
if (out == NULL) return(-1);
retry:
written = out->size - out->use;
if (written > 0)
written--; /* Gennady: count '/0' */
/*
* First specific handling of in = NULL, i.e. the initialization call
*/
if (in == NULL) {
toconv = 0;
/* TODO: Check return value. */
xmlEncOutputChunk(handler, &out->content[out->use], &written,
NULL, &toconv);
out->use += written;
out->content[out->use] = 0;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"initialized encoder\n");
#endif
return(0);
}
/*
* Conversion itself.
*/
toconv = in->use;
if (toconv == 0)
return(0);
if (toconv * 4 >= written) {
xmlBufferGrow(out, toconv * 4);
written = out->size - out->use - 1;
}
ret = xmlEncOutputChunk(handler, &out->content[out->use], &written,
in->content, &toconv);
xmlBufferShrink(in, toconv);
out->use += written;
writtentot += written;
out->content[out->use] = 0;
if (ret == -1) {
if (written > 0) {
/* Can be a limitation of iconv or uconv */
goto retry;
}
ret = -3;
}
if (ret >= 0) output += ret;
/*
* Attempt to handle error cases
*/
switch (ret) {
case 0:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"converted %d bytes to %d bytes of output\n",
toconv, written);
#endif
break;
case -1:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"output conversion failed by lack of space\n");
#endif
break;
case -3:
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,"converted %d bytes to %d bytes of output %d left\n",
toconv, written, in->use);
#endif
break;
case -4:
xmlEncodingErr(XML_I18N_NO_OUTPUT,
"xmlCharEncOutFunc: no output function !\n", NULL);
ret = -1;
break;
case -2: {
xmlChar charref[20];
int len = in->use;
const xmlChar *utf = (const xmlChar *) in->content;
int cur, charrefLen;
cur = xmlGetUTF8Char(utf, &len);
if (cur <= 0)
break;
#ifdef DEBUG_ENCODING
xmlGenericError(xmlGenericErrorContext,
"handling output conversion error\n");
xmlGenericError(xmlGenericErrorContext,
"Bytes: 0x%02X 0x%02X 0x%02X 0x%02X\n",
in->content[0], in->content[1],
in->content[2], in->content[3]);
#endif
/*
* Removes the UTF8 sequence, and replace it by a charref
* and continue the transcoding phase, hoping the error
* did not mangle the encoder state.
*/
charrefLen = snprintf((char *) &charref[0], sizeof(charref),
"&#%d;", cur);
xmlBufferShrink(in, len);
xmlBufferGrow(out, charrefLen * 4);
written = out->size - out->use - 1;
toconv = charrefLen;
ret = xmlEncOutputChunk(handler, &out->content[out->use], &written,
charref, &toconv);
if ((ret < 0) || (toconv != charrefLen)) {
char buf[50];
snprintf(&buf[0], 49, "0x%02X 0x%02X 0x%02X 0x%02X",
in->content[0], in->content[1],
in->content[2], in->content[3]);
buf[49] = 0;
xmlEncodingErr(XML_I18N_CONV_FAILED,
"output conversion failed due to conv error, bytes %s\n",
buf);
if (in->alloc != XML_BUFFER_ALLOC_IMMUTABLE)
in->content[0] = ' ';
break;
}
out->use += written;
writtentot += written;
out->content[out->use] = 0;
goto retry;
}
}
return(writtentot ? writtentot : ret);
}
/**
* xmlCharEncCloseFunc:
* @handler: char encoding transformation data structure
*
* Generic front-end for encoding handler close function
*
* Returns 0 if success, or -1 in case of error
*/
int
xmlCharEncCloseFunc(xmlCharEncodingHandler *handler) {
int ret = 0;
int tofree = 0;
int i, handler_in_list = 0;
if (handler == NULL) return(-1);
if (handler->name == NULL) return(-1);
if (handlers != NULL) {
for (i = 0;i < nbCharEncodingHandler; i++) {
if (handler == handlers[i]) {
handler_in_list = 1;
break;
}
}
}
#ifdef LIBXML_ICONV_ENABLED
/*
* Iconv handlers can be used only once, free the whole block.
* and the associated icon resources.
*/
if ((handler_in_list == 0) &&
((handler->iconv_out != NULL) || (handler->iconv_in != NULL))) {
tofree = 1;
if (handler->iconv_out != NULL) {
if (iconv_close(handler->iconv_out))
ret = -1;
handler->iconv_out = NULL;
}
if (handler->iconv_in != NULL) {
if (iconv_close(handler->iconv_in))
ret = -1;
handler->iconv_in = NULL;
}
}
#endif /* LIBXML_ICONV_ENABLED */
#ifdef LIBXML_ICU_ENABLED
if ((handler_in_list == 0) &&
((handler->uconv_out != NULL) || (handler->uconv_in != NULL))) {
tofree = 1;
if (handler->uconv_out != NULL) {
closeIcuConverter(handler->uconv_out);
handler->uconv_out = NULL;
}
if (handler->uconv_in != NULL) {
closeIcuConverter(handler->uconv_in);
handler->uconv_in = NULL;
}
}
#endif
if (tofree) {
/* free up only dynamic handlers iconv/uconv */
if (handler->name != NULL)
xmlFree(handler->name);
handler->name = NULL;
xmlFree(handler);
}
#ifdef DEBUG_ENCODING
if (ret)
xmlGenericError(xmlGenericErrorContext,
"failed to close the encoding handler\n");
else
xmlGenericError(xmlGenericErrorContext,
"closed the encoding handler\n");
#endif
return(ret);
}
/**
* xmlByteConsumed:
* @ctxt: an XML parser context
*
* This function provides the current index of the parser relative
* to the start of the current entity. This function is computed in
* bytes from the beginning starting at zero and finishing at the
* size in byte of the file if parsing a file. The function is
* of constant cost if the input is UTF-8 but can be costly if run
* on non-UTF-8 input.
*
* Returns the index in bytes from the beginning of the entity or -1
* in case the index could not be computed.
*/
long
xmlByteConsumed(xmlParserCtxtPtr ctxt) {
xmlParserInputPtr in;
if (ctxt == NULL) return(-1);
in = ctxt->input;
if (in == NULL) return(-1);
if ((in->buf != NULL) && (in->buf->encoder != NULL)) {
unsigned int unused = 0;
xmlCharEncodingHandler * handler = in->buf->encoder;
/*
* Encoding conversion, compute the number of unused original
* bytes from the input not consumed and subtract that from
* the raw consumed value, this is not a cheap operation
*/
if (in->end - in->cur > 0) {
unsigned char convbuf[32000];
const unsigned char *cur = (const unsigned char *)in->cur;
int toconv = in->end - in->cur, written = 32000;
int ret;
do {
toconv = in->end - cur;
written = 32000;
ret = xmlEncOutputChunk(handler, &convbuf[0], &written,
cur, &toconv);
if (ret < 0) {
if (written > 0)
ret = -2;
else
return(-1);
}
unused += written;
cur += toconv;
} while (ret == -2);
}
if (in->buf->rawconsumed < unused)
return(-1);
return(in->buf->rawconsumed - unused);
}
return(in->consumed + (in->cur - in->base));
}
#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED)
#ifdef LIBXML_ISO8859X_ENABLED
/**
* UTF8ToISO8859x:
* @out: a pointer to an array of bytes to store the result
* @outlen: the length of @out
* @in: a pointer to an array of UTF-8 chars
* @inlen: the length of @in
* @xlattable: the 2-level transcoding table
*
* Take a block of UTF-8 chars in and try to convert it to an ISO 8859-*
* block of chars out.
*
* Returns 0 if success, -2 if the transcoding fails, or -1 otherwise
* The value of @inlen after return is the number of octets consumed
* as the return value is positive, else unpredictable.
* The value of @outlen after return is the number of octets consumed.
*/
static int
UTF8ToISO8859x(unsigned char* out, int *outlen,
const unsigned char* in, int *inlen,
unsigned char const *xlattable) {
const unsigned char* outstart = out;
const unsigned char* inend;
const unsigned char* instart = in;
const unsigned char* processed = in;
if ((out == NULL) || (outlen == NULL) || (inlen == NULL) ||
(xlattable == NULL))
return(-1);
if (in == NULL) {
/*
* initialization nothing to do
*/
*outlen = 0;
*inlen = 0;
return(0);
}
inend = in + (*inlen);
while (in < inend) {
unsigned char d = *in++;
if (d < 0x80) {
*out++ = d;
} else if (d < 0xC0) {
/* trailing byte in leading position */
*outlen = out - outstart;
*inlen = processed - instart;
return(-2);
} else if (d < 0xE0) {
unsigned char c;
if (!(in < inend)) {
/* trailing byte not in input buffer */
*outlen = out - outstart;
*inlen = processed - instart;
return(-3);
}
c = *in++;
if ((c & 0xC0) != 0x80) {
/* not a trailing byte */
*outlen = out - outstart;
*inlen = processed - instart;