diff options
Diffstat (limited to '')
| -rw-r--r-- | libutf8.h | 351 |
1 files changed, 351 insertions, 0 deletions
diff --git a/libutf8.h b/libutf8.h new file mode 100644 index 0000000..8feeebf --- /dev/null +++ b/libutf8.h @@ -0,0 +1,351 @@ +/* See LICENSE file for copyright and license details. */ +#ifndef LIBUTF8_H +#define LIBUTF8_H + +#include <stddef.h> +#include <stdint.h> + +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Winline" +#endif + + +/** + * The greatest Unicode codepoint + * + * @seealso libutf8_is_invalid + */ +#define LIBUTF8_ULTIMATE_CODEPOINT UINT32_C(0x10FFFF) + + +/** + * The Unicode replacement character as a UTF-32 codepoint + * + * @seealso LIBUTF8_REPLACEMENT_CHARACTER_UTF8 + */ +#define LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT 0xFFFDu + + +/** + * The Unicode replacement character encoded as UTF-8 + * + * @seealso LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT + */ +#define LIBUTF8_REPLACEMENT_CHARACTER_UTF8 "\xEF\xBF\xBD" + + +/** + * Replace a decoded invalid codepoint with + * `LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT` + * + * @seealso LIBUTF8_RETURN_INVALID + */ +#define LIBUTF8_REPLACE_INVALID 0x0001u + + +/** + * Return a decoded invalid codepoint unchanged + * + * @seealso LIBUTF8_REPLACE_INVALID + */ +#define LIBUTF8_RETURN_INVALID 0x0002u + + +/** + * Accept UTF-8 sequences that decode past + * `LIBUTF8_ULTIMATE_CODEPOINT` to a maximum of 31 bits + */ +#define LIBUTF8_ALLOW_31_BITS 0x0004u + + +/** + * Accept the two-byte overlong encoding of U+0000 + * + * @seealso LIBUTF8_ALLOW_LONG_ENCODING + */ +#define LIBUTF8_ALLOW_2BYTE_NULL 0x0008u + + +/** + * Accept all overlong UTF-8 encodings + * + * This flag implies LIBUTF8_ALLOW_2BYTE_NULL + * + * @seealso LIBUTF8_ALLOW_2BYTE_NULL + */ +#define LIBUTF8_ALLOW_LONG_ENCODING 0x0010u + + +/** + * Test whether a codepoint is a non-character Unicode codepoint + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is a non-character Unicode codepoint, 0 otherwise + * + * A non-character Unicode codepoint is codepoint + * reserved as invalid and without special meaning + * + * The result is unspecified if `codepoint` is + * greater than LIBUTF8_ULTIMATE_CODEPOINT + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_noncharacter(uint_least32_t codepoint) +{ + return (0xFDD0u <= codepoint && codepoint <= 0xFDEFu) || (codepoint & 0xFFFEu) == 0xFFFEu; +} + + +/** + * Test whether a codepoint is a UTF-16 surrogate + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is a UTF-16 surrogate, 0 otherwise + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_surrogate(uint_least32_t codepoint) +{ + return 0xD800u <= codepoint && codepoint <= 0xDFFFu; +} + + +/** + * Test whether a codepoint is invalid Unicode + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is outside the Unicode range, + * a surrogate, or a non-character; 0 otherwise + * + * @seealso libutf8_is_valid + * @seealso LIBUTF8_ULTIMATE_CODEPOINT + * @seealso libutf8_is_noncharacter + * @seealso libutf8_is_surrogate + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_invalid(uint_least32_t codepoint) +{ + return codepoint > LIBUTF8_ULTIMATE_CODEPOINT || + libutf8_is_noncharacter(codepoint) || + libutf8_is_surrogate(codepoint); +} + + +/** + * Test whether a codepoint is valid Unicode + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is within the Unicode range + * and is neither a surrogate nor a non-character, + * 0 otherwise + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_valid(uint_least32_t codepoint) +{ + return !libutf8_is_invalid(codepoint); +} + + +/** + * Test whether a byte is a UTF-8 continuation byte + * + * @param c The byte to test + * @return 1 if `c` is a codepoint continuation byte, + * 0 otherwise + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_continuation(char c) +{ + return ((unsigned char)c & 0xC0u) == 0x80u; +} + + +/** + * Get the number of bytes required to encode a codepoint + * + * @param codepoint The codepoint to measure + * @return The number of bytes required to encode `codepoint`; + * 0 if `codepoint` does not fit in an unsigned 31-bit integer + * + * The maximum return value is 4 for valid Unicode + * codepoints and 6 for any value accept by `libutf8_encode` + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EDOM` when this + * function returns 0 + * + * @seealso libutf8_encode + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline size_t +libutf8_encode_measure(uint_least32_t codepoint) +{ + if (codepoint >> 7 == 0u) return 1u; + if (codepoint >> 11 == 0u) return 2u; + if (codepoint >> 16 == 0u) return 3u; + if (codepoint >> 21 == 0u) return 4u; + if (codepoint >> 26 == 0u) return 5u; + if (codepoint >> 31 == 0u) return 6u; + return 0u; +} + + +/** + * Encode a codepoint as UTF-8 + * + * @param out_buffer Output buffer; must have room for the number + * of bytes required to encode `codepoint` + * @param codepoint The codepoint to encode + * @return The number of bytes written + * + * NB! The output is not NUL-terminated + * + * Besides Unicode codepoints, this function encodes 31-bit + * values greater than `LIBUTF8_ULTIMATE_CODEPOINT` using + * historical UTF-8 sequences up to 6 bytes. This lets + * applications embed application-specific data in a file. + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EDOM` when this + * function returns 0 + * + * @seealso libutf8_encode_measure + * @seealso libutf8_is_valid + * @seealso libutf8_decode + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((nonnull(1), warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((nonnull(1), access(write_only, 1), warn_unused_result)) +#endif +size_t libutf8_encode(char *out_buffer, uint_least32_t codepoint); + + +/** + * Decode the first UTF-8 sequence in a string + * + * @param text The input string; must contain a complete candidate sequence + * @param codepoint_out Unless `NULL`, output for the decoded codepoint + * @param flags Bitwise OR of zero or more or the following flags: + * - LIBUTF8_REPLACE_INVALID: + * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, + * with the REPLACEMENT CHARACTER. + * NB! This only affects invalid codepoints, not illegal byte sequences. + * - LIBUTF8_RETURN_INVALID: + * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid + * - LIBUTF8_ALLOW_31_BITS + * Treat 31-bit codepoints greater than + * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. + * - LIBUTF8_ALLOW_2BYTE_NULL + * Accept the overlong 2-byte encoding of U+0000. + * - LIBUTF8_ALLOW_LONG_ENCODING + * Accept overlong encodings, up to 6 bytes. + * @return The number of decoded bytes, 0 if the input is invalid + * + * It is unspecified which flag dominates: + * LIBUTF8_REPLACE_INVALID or LIBUTF8_RETURN_INVALID + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EILSEQ` when this + * function returns 0 + * + * @seealso libutf8_decode_mem + * @seealso libutf8_is_continuation + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((nonnull(1), warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((nonnull(1), access(write_only, 2), warn_unused_result)) +#endif +size_t libutf8_decode(const char *text, uint_least32_t *codepoint_out, unsigned flags); + + +/** + * Decode the first UTF-8 sequence in a string + * + * @param text The input string; must contain a complete candidate sequence + * @param text_len The number of bytes available in `text` + * @param codepoint_out Unless `NULL`, output for the decoded codepoint + * @param flags Bitwise OR of zero or more or the following flags: + * - LIBUTF8_REPLACE_INVALID: + * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, + * with the REPLACEMENT CHARACTER. + * NB! This only affects invalid codepoints, not illegal byte sequences. + * - LIBUTF8_RETURN_INVALID: + * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid + * - LIBUTF8_ALLOW_31_BITS + * Treat 31-bit codepoints greater than + * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. + * - LIBUTF8_ALLOW_2BYTE_NULL + * Accept the overlong 2-byte encoding of U+0000. + * - LIBUTF8_ALLOW_LONG_ENCODING + * Accept overlong encodings, up to 6 bytes. + * @return The number of decoded bytes, 0 if the input is invalid; + * or the required sequence length specified by a valid + * leading byte if `text_len` is less than this value + * (input is truncated) + * + * Note, that a return value greater than `text_len` does not + * necessarily (it is unspecified) indicate that all available + * bytes are invalid, only that the first byte is valid + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EILSEQ` when this + * function returns 0 + * + * @seealso libutf8_decode + * @seealso libutf8_is_continuation + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((access(read_only, 1, 2), access(write_only, 3), warn_unused_result)) +#endif +size_t libutf8_decode_mem(const char *text, size_t text_len, uint_least32_t *codepoint_out, unsigned flags); + + +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif + +#endif |