Commit 10ecb82e authored by Lars Schneider's avatar Lars Schneider Committed by Junio C Hamano

utf8: add function to detect prohibited UTF-16/32 BOM

Whenever a data stream is declared to be UTF-16BE, UTF-16LE, UTF-32BE
or UTF-32LE a BOM must not be used [1]. The function returns true if
this is the case.

This function is used in a subsequent commit.

[1] http://unicode.org/faq/utf_bom.html#bom10Signed-off-by: default avatarLars Schneider <larsxschneider@gmail.com>
Signed-off-by: default avatarJunio C Hamano <gitster@pobox.com>
parent 2f0c4a36
......@@ -560,6 +560,32 @@ char *reencode_string_len(const char *in, int insz,
}
#endif
static int has_bom_prefix(const char *data, size_t len,
const char *bom, size_t bom_len)
{
return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len);
}
static const char utf16_be_bom[] = {0xFE, 0xFF};
static const char utf16_le_bom[] = {0xFF, 0xFE};
static const char utf32_be_bom[] = {0x00, 0x00, 0xFE, 0xFF};
static const char utf32_le_bom[] = {0xFF, 0xFE, 0x00, 0x00};
int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
{
return (
(same_utf_encoding("UTF-16BE", enc) ||
same_utf_encoding("UTF-16LE", enc)) &&
(has_bom_prefix(data, len, utf16_be_bom, sizeof(utf16_be_bom)) ||
has_bom_prefix(data, len, utf16_le_bom, sizeof(utf16_le_bom)))
) || (
(same_utf_encoding("UTF-32BE", enc) ||
same_utf_encoding("UTF-32LE", enc)) &&
(has_bom_prefix(data, len, utf32_be_bom, sizeof(utf32_be_bom)) ||
has_bom_prefix(data, len, utf32_le_bom, sizeof(utf32_le_bom)))
);
}
/*
* Returns first character length in bytes for multi-byte `text` according to
* `encoding`.
......
......@@ -70,4 +70,13 @@ typedef enum {
void strbuf_utf8_align(struct strbuf *buf, align_type position, unsigned int width,
const char *s);
/*
* If a data stream is declared as UTF-16BE or UTF-16LE, then a UTF-16
* BOM must not be used [1]. The same applies for the UTF-32 equivalents.
* The function returns true if this rule is violated.
*
* [1] http://unicode.org/faq/utf_bom.html#bom10
*/
int has_prohibited_utf_bom(const char *enc, const char *data, size_t len);
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment