/* See LICENSE file for copyright and license details. */ #ifndef LIBUTF8_H #define LIBUTF8_H #include #include #if defined(__GNUC__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Winline" #endif /** * The greatest Unicode codepoint * * @seealso libutf8_is_invalid */ #define LIBUTF8_ULTIMATE_CODEPOINT UINT32_C(0x10FFFF) /** * The Unicode replacement character as a UTF-32 codepoint * * @seealso LIBUTF8_REPLACEMENT_CHARACTER_UTF8 */ #define LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT 0xFFFDu /** * The Unicode replacement character encoded as UTF-8 * * @seealso LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT */ #define LIBUTF8_REPLACEMENT_CHARACTER_UTF8 "\xEF\xBF\xBD" /** * Replace a decoded invalid codepoint with * `LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT` * * @seealso LIBUTF8_RETURN_INVALID */ #define LIBUTF8_REPLACE_INVALID 0x0001u /** * Return a decoded invalid codepoint unchanged * * @seealso LIBUTF8_REPLACE_INVALID */ #define LIBUTF8_RETURN_INVALID 0x0002u /** * Accept UTF-8 sequences that decode past * `LIBUTF8_ULTIMATE_CODEPOINT` to a maximum of 31 bits */ #define LIBUTF8_ALLOW_31_BITS 0x0004u /** * Accept the two-byte overlong encoding of U+0000 * * @seealso LIBUTF8_ALLOW_LONG_ENCODING */ #define LIBUTF8_ALLOW_2BYTE_NULL 0x0008u /** * Accept all overlong UTF-8 encodings * * This flag implies LIBUTF8_ALLOW_2BYTE_NULL * * @seealso LIBUTF8_ALLOW_2BYTE_NULL */ #define LIBUTF8_ALLOW_LONG_ENCODING 0x0010u /** * Test whether a codepoint is a non-character Unicode codepoint * * @param codepoint The codepoint to test * @return 1 if `codepoint` is a non-character Unicode codepoint, 0 otherwise * * A non-character Unicode codepoint is codepoint * reserved as invalid and without special meaning * * The result is unspecified if `codepoint` is * greater than LIBUTF8_ULTIMATE_CODEPOINT * * @seealso libutf8_is_invalid * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline int libutf8_is_noncharacter(uint_least32_t codepoint) { return (0xFDD0u <= codepoint && codepoint <= 0xFDEFu) || (codepoint & 0xFFFEu) == 0xFFFEu; } /** * Test whether a codepoint is a UTF-16 surrogate * * @param codepoint The codepoint to test * @return 1 if `codepoint` is a UTF-16 surrogate, 0 otherwise * * @seealso libutf8_is_invalid * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline int libutf8_is_surrogate(uint_least32_t codepoint) { return 0xD800u <= codepoint && codepoint <= 0xDFFFu; } /** * Test whether a codepoint is invalid Unicode * * @param codepoint The codepoint to test * @return 1 if `codepoint` is outside the Unicode range, * a surrogate, or a non-character; 0 otherwise * * @seealso libutf8_is_valid * @seealso LIBUTF8_ULTIMATE_CODEPOINT * @seealso libutf8_is_noncharacter * @seealso libutf8_is_surrogate * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline int libutf8_is_invalid(uint_least32_t codepoint) { return codepoint > LIBUTF8_ULTIMATE_CODEPOINT || libutf8_is_noncharacter(codepoint) || libutf8_is_surrogate(codepoint); } /** * Test whether a codepoint is valid Unicode * * @param codepoint The codepoint to test * @return 1 if `codepoint` is within the Unicode range * and is neither a surrogate nor a non-character, * 0 otherwise * * @seealso libutf8_is_invalid * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline int libutf8_is_valid(uint_least32_t codepoint) { return !libutf8_is_invalid(codepoint); } /** * Test whether a byte is a UTF-8 continuation byte * * @param c The byte to test * @return 1 if `c` is a codepoint continuation byte, * 0 otherwise * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline int libutf8_is_continuation(char c) { return ((unsigned char)c & 0xC0u) == 0x80u; } /** * Get the number of bytes required to encode a codepoint * * @param codepoint The codepoint to measure * @return The number of bytes required to encode `codepoint`; * 0 if `codepoint` does not fit in an unsigned 31-bit integer * * The maximum return value is 4 for valid Unicode * codepoints and 6 for any value accept by `libutf8_encode` * * This function never sets `errno`, however an * appropriate `errno`-value is `EDOM` when this * function returns 0 * * @seealso libutf8_encode * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__GNUC__) __attribute__((const, warn_unused_result)) #endif inline size_t libutf8_encode_measure(uint_least32_t codepoint) { if (codepoint >> 7 == 0u) return 1u; if (codepoint >> 11 == 0u) return 2u; if (codepoint >> 16 == 0u) return 3u; if (codepoint >> 21 == 0u) return 4u; if (codepoint >> 26 == 0u) return 5u; if (codepoint >> 31 == 0u) return 6u; return 0u; } /** * Encode a codepoint as UTF-8 * * @param out_buffer Output buffer; must have room for the number * of bytes required to encode `codepoint` * @param codepoint The codepoint to encode * @return The number of bytes written * * NB! The output is not NUL-terminated * * Besides Unicode codepoints, this function encodes 31-bit * values greater than `LIBUTF8_ULTIMATE_CODEPOINT` using * historical UTF-8 sequences up to 6 bytes. This lets * applications embed application-specific data in a file. * * This function never sets `errno`, however an * appropriate `errno`-value is `EDOM` when this * function returns 0 * * @seealso libutf8_encode_measure * @seealso libutf8_is_valid * @seealso libutf8_decode * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__clang__) __attribute__((nonnull(1), warn_unused_result)) #elif defined(__GNUC__) __attribute__((nonnull(1), access(write_only, 1), warn_unused_result)) #endif size_t libutf8_encode(char *out_buffer, uint_least32_t codepoint); /** * Decode the first UTF-8 sequence in a string * * @param text The input string; must contain a complete candidate sequence * @param codepoint_out Unless `NULL`, output for the decoded codepoint * @param flags Bitwise OR of zero or more or the following flags: * - LIBUTF8_REPLACE_INVALID: * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, * with the REPLACEMENT CHARACTER. * NB! This only affects invalid codepoints, not illegal byte sequences. * - LIBUTF8_RETURN_INVALID: * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid * - LIBUTF8_ALLOW_31_BITS * Treat 31-bit codepoints greater than * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. * - LIBUTF8_ALLOW_2BYTE_NULL * Accept the overlong 2-byte encoding of U+0000. * - LIBUTF8_ALLOW_LONG_ENCODING * Accept overlong encodings, up to 6 bytes. * @return The number of decoded bytes, 0 if the input is invalid * * It is unspecified which flag dominates: * LIBUTF8_REPLACE_INVALID or LIBUTF8_RETURN_INVALID * * This function never sets `errno`, however an * appropriate `errno`-value is `EILSEQ` when this * function returns 0 * * @seealso libutf8_decode_mem * @seealso libutf8_is_continuation * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__clang__) __attribute__((nonnull(1), warn_unused_result)) #elif defined(__GNUC__) __attribute__((nonnull(1), access(write_only, 2), warn_unused_result)) #endif size_t libutf8_decode(const char *text, uint_least32_t *codepoint_out, unsigned flags); /** * Decode the first UTF-8 sequence in a string * * @param text The input string; must contain a complete candidate sequence * @param text_len The number of bytes available in `text` * @param codepoint_out Unless `NULL`, output for the decoded codepoint * @param flags Bitwise OR of zero or more or the following flags: * - LIBUTF8_REPLACE_INVALID: * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, * with the REPLACEMENT CHARACTER. * NB! This only affects invalid codepoints, not illegal byte sequences. * - LIBUTF8_RETURN_INVALID: * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid * - LIBUTF8_ALLOW_31_BITS * Treat 31-bit codepoints greater than * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. * - LIBUTF8_ALLOW_2BYTE_NULL * Accept the overlong 2-byte encoding of U+0000. * - LIBUTF8_ALLOW_LONG_ENCODING * Accept overlong encodings, up to 6 bytes. * @return The number of decoded bytes, 0 if the input is invalid; * or the required sequence length specified by a valid * leading byte if `text_len` is less than this value * (input is truncated) * * Note, that a return value greater than `text_len` does not * necessarily (it is unspecified) indicate that all available * bytes are invalid, only that the first byte is valid * * This function never sets `errno`, however an * appropriate `errno`-value is `EILSEQ` when this * function returns 0 * * @seealso libutf8_decode * @seealso libutf8_is_continuation * * This function is MT-Safe, AS-Safe, and AC-Safe */ #if defined(__clang__) __attribute__((warn_unused_result)) #elif defined(__GNUC__) __attribute__((access(read_only, 1, 2), access(write_only, 3), warn_unused_result)) #endif size_t libutf8_decode_mem(const char *text, size_t text_len, uint_least32_t *codepoint_out, unsigned flags); #if defined(__GNUC__) # pragma GCC diagnostic pop #endif #endif