First commitHEAD1.0master

Signed-off-by: Mattias Andrée <m@maandree.se>

1 files changed, 351 insertions, 0 deletions

diff --git a/libutf8.h b/libutf8.h
new file mode 100644
index 0000000..8feeebf
--- /dev/null
+++ b/libutf8.h
@@ -0,0 +1,351 @@
+/* See LICENSE file for copyright and license details. */
+#ifndef LIBUTF8_H
+#define LIBUTF8_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__GNUC__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Winline"
+#endif
+
+
+/**
+ * The greatest Unicode codepoint
+ * 
+ * @seealso  libutf8_is_invalid
+ */
+#define LIBUTF8_ULTIMATE_CODEPOINT UINT32_C(0x10FFFF)
+
+
+/**
+ * The Unicode replacement character as a UTF-32 codepoint
+ * 
+ * @seealso  LIBUTF8_REPLACEMENT_CHARACTER_UTF8
+ */
+#define LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT 0xFFFDu
+
+
+/**
+ * The Unicode replacement character encoded as UTF-8
+ * 
+ * @seealso  LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT
+ */
+#define LIBUTF8_REPLACEMENT_CHARACTER_UTF8 "\xEF\xBF\xBD"
+
+
+/**
+ * Replace a decoded invalid codepoint with
+ * `LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT`
+ * 
+ * @seealso  LIBUTF8_RETURN_INVALID
+ */
+#define LIBUTF8_REPLACE_INVALID 0x0001u
+
+
+/**
+ * Return a decoded invalid codepoint unchanged
+ * 
+ * @seealso  LIBUTF8_REPLACE_INVALID
+ */
+#define LIBUTF8_RETURN_INVALID 0x0002u
+
+
+/**
+ * Accept UTF-8 sequences that decode past
+ * `LIBUTF8_ULTIMATE_CODEPOINT` to a maximum of 31 bits
+ */
+#define LIBUTF8_ALLOW_31_BITS 0x0004u
+
+
+/**
+ * Accept the two-byte overlong encoding of U+0000
+ * 
+ * @seealso  LIBUTF8_ALLOW_LONG_ENCODING
+ */
+#define LIBUTF8_ALLOW_2BYTE_NULL 0x0008u
+
+
+/**
+ * Accept all overlong UTF-8 encodings
+ * 
+ * This flag implies LIBUTF8_ALLOW_2BYTE_NULL
+ * 
+ * @seealso  LIBUTF8_ALLOW_2BYTE_NULL
+ */
+#define LIBUTF8_ALLOW_LONG_ENCODING 0x0010u
+
+
+/**
+ * Test whether a codepoint is a non-character Unicode codepoint
+ * 
+ * @param   codepoint  The codepoint to test
+ * @return             1 if `codepoint` is a non-character Unicode codepoint, 0 otherwise
+ * 
+ * A non-character Unicode codepoint is codepoint
+ * reserved as invalid and without special meaning
+ * 
+ * The result is unspecified if `codepoint` is
+ * greater than LIBUTF8_ULTIMATE_CODEPOINT
+ * 
+ * @seealso  libutf8_is_invalid
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline int
+libutf8_is_noncharacter(uint_least32_t codepoint)
+{
+	return (0xFDD0u <= codepoint && codepoint <= 0xFDEFu) || (codepoint & 0xFFFEu) == 0xFFFEu;
+}
+
+
+/**
+ * Test whether a codepoint is a UTF-16 surrogate
+ * 
+ * @param   codepoint  The codepoint to test
+ * @return             1 if `codepoint` is a UTF-16 surrogate, 0 otherwise
+ * 
+ * @seealso  libutf8_is_invalid
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline int
+libutf8_is_surrogate(uint_least32_t codepoint)
+{
+	return 0xD800u <= codepoint && codepoint <= 0xDFFFu;
+}
+
+
+/**
+ * Test whether a codepoint is invalid Unicode
+ * 
+ * @param   codepoint  The codepoint to test
+ * @return             1 if `codepoint` is outside the Unicode range,
+ *                     a surrogate, or a non-character; 0 otherwise
+ * 
+ * @seealso  libutf8_is_valid
+ * @seealso  LIBUTF8_ULTIMATE_CODEPOINT
+ * @seealso  libutf8_is_noncharacter
+ * @seealso  libutf8_is_surrogate
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline int
+libutf8_is_invalid(uint_least32_t codepoint)
+{
+	return codepoint > LIBUTF8_ULTIMATE_CODEPOINT ||
+	       libutf8_is_noncharacter(codepoint) ||
+	       libutf8_is_surrogate(codepoint);
+}
+
+
+/**
+ * Test whether a codepoint is valid Unicode
+ * 
+ * @param   codepoint  The codepoint to test
+ * @return             1 if `codepoint` is within the Unicode range
+ *                     and is neither a surrogate nor a non-character,
+ *                     0 otherwise
+ * 
+ * @seealso  libutf8_is_invalid
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline int
+libutf8_is_valid(uint_least32_t codepoint)
+{
+	return !libutf8_is_invalid(codepoint);
+}
+
+
+/**
+ * Test whether a byte is a UTF-8 continuation byte
+ * 
+ * @param   c  The byte to test
+ * @return     1 if `c` is a codepoint continuation byte,
+ *             0 otherwise
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline int
+libutf8_is_continuation(char c)
+{
+	return ((unsigned char)c & 0xC0u) == 0x80u;
+}
+
+
+/**
+ * Get the number of bytes required to encode a codepoint
+ * 
+ * @param   codepoint  The codepoint to measure
+ * @return             The number of bytes required to encode `codepoint`;
+ *                     0 if `codepoint` does not fit in an unsigned 31-bit integer
+ * 
+ * The maximum return value is 4 for valid Unicode
+ * codepoints and 6 for any value accept by `libutf8_encode`
+ * 
+ * This function never sets `errno`, however an
+ * appropriate `errno`-value is `EDOM` when this
+ * function returns 0
+ * 
+ * @seealso  libutf8_encode
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__GNUC__)
+__attribute__((const, warn_unused_result))
+#endif
+inline size_t
+libutf8_encode_measure(uint_least32_t codepoint)
+{
+	if (codepoint >>  7 == 0u) return 1u;
+	if (codepoint >> 11 == 0u) return 2u;
+	if (codepoint >> 16 == 0u) return 3u;
+	if (codepoint >> 21 == 0u) return 4u;
+	if (codepoint >> 26 == 0u) return 5u;
+	if (codepoint >> 31 == 0u) return 6u;
+	return 0u;
+}
+
+
+/**
+ * Encode a codepoint as UTF-8
+ * 
+ * @param   out_buffer  Output buffer; must have room for the number
+ *                      of bytes required to encode `codepoint`
+ * @param   codepoint   The codepoint to encode
+ * @return              The number of bytes written
+ * 
+ * NB! The output is not NUL-terminated
+ * 
+ * Besides Unicode codepoints, this function encodes 31-bit
+ * values greater than `LIBUTF8_ULTIMATE_CODEPOINT` using
+ * historical UTF-8 sequences up to 6 bytes. This lets
+ * applications embed application-specific data in a file.
+ * 
+ * This function never sets `errno`, however an
+ * appropriate `errno`-value is `EDOM` when this
+ * function returns 0
+ * 
+ * @seealso  libutf8_encode_measure
+ * @seealso  libutf8_is_valid
+ * @seealso  libutf8_decode
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__clang__)
+__attribute__((nonnull(1), warn_unused_result))
+#elif defined(__GNUC__)
+__attribute__((nonnull(1), access(write_only, 1), warn_unused_result))
+#endif
+size_t libutf8_encode(char *out_buffer, uint_least32_t codepoint);
+
+
+/**
+ * Decode the first UTF-8 sequence in a string
+ * 
+ * @param   text           The input string; must contain a complete candidate sequence
+ * @param   codepoint_out  Unless `NULL`, output for the decoded codepoint
+ * @param   flags          Bitwise OR of zero or more or the following flags:
+ *                         -  LIBUTF8_REPLACE_INVALID:
+ *                                Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`,
+ *                                with the REPLACEMENT CHARACTER.
+ *                                NB! This only affects invalid codepoints, not illegal byte sequences.
+ *                         -  LIBUTF8_RETURN_INVALID:
+ *                                Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid
+ *                         -  LIBUTF8_ALLOW_31_BITS
+ *                                Treat 31-bit codepoints greater than
+ *                                `LIBUTF8_ULTIMATE_CODEPOINT` as valid.
+ *                         -  LIBUTF8_ALLOW_2BYTE_NULL
+ *                                Accept the overlong 2-byte encoding of U+0000.
+ *                         -  LIBUTF8_ALLOW_LONG_ENCODING
+ *                                Accept overlong encodings, up to 6 bytes.
+ * @return                 The number of decoded bytes, 0 if the input is invalid
+ * 
+ * It is unspecified which flag dominates:
+ * LIBUTF8_REPLACE_INVALID or LIBUTF8_RETURN_INVALID
+ * 
+ * This function never sets `errno`, however an
+ * appropriate `errno`-value is `EILSEQ` when this
+ * function returns 0
+ * 
+ * @seealso  libutf8_decode_mem
+ * @seealso  libutf8_is_continuation
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__clang__)
+__attribute__((nonnull(1), warn_unused_result))
+#elif defined(__GNUC__)
+__attribute__((nonnull(1), access(write_only, 2), warn_unused_result))
+#endif
+size_t libutf8_decode(const char *text, uint_least32_t *codepoint_out, unsigned flags);
+
+
+/**
+ * Decode the first UTF-8 sequence in a string
+ * 
+ * @param   text           The input string; must contain a complete candidate sequence
+ * @param   text_len       The number of bytes available in `text`
+ * @param   codepoint_out  Unless `NULL`, output for the decoded codepoint
+ * @param   flags          Bitwise OR of zero or more or the following flags:
+ *                         -  LIBUTF8_REPLACE_INVALID:
+ *                                Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`,
+ *                                with the REPLACEMENT CHARACTER.
+ *                                NB! This only affects invalid codepoints, not illegal byte sequences.
+ *                         -  LIBUTF8_RETURN_INVALID:
+ *                                Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid
+ *                         -  LIBUTF8_ALLOW_31_BITS
+ *                                Treat 31-bit codepoints greater than
+ *                                `LIBUTF8_ULTIMATE_CODEPOINT` as valid.
+ *                         -  LIBUTF8_ALLOW_2BYTE_NULL
+ *                                Accept the overlong 2-byte encoding of U+0000.
+ *                         -  LIBUTF8_ALLOW_LONG_ENCODING
+ *                                Accept overlong encodings, up to 6 bytes.
+ * @return                 The number of decoded bytes, 0 if the input is invalid;
+ *                         or the required sequence length specified by a valid
+ *                         leading byte if `text_len` is less than this value
+ *                         (input is truncated)
+ *
+ * Note, that a return value greater than `text_len` does not
+ * necessarily (it is unspecified) indicate that all available
+ * bytes are invalid, only that the first byte is valid
+ * 
+ * This function never sets `errno`, however an
+ * appropriate `errno`-value is `EILSEQ` when this
+ * function returns 0
+ * 
+ * @seealso  libutf8_decode
+ * @seealso  libutf8_is_continuation
+ * 
+ * This function is MT-Safe, AS-Safe, and AC-Safe
+ */
+#if defined(__clang__)
+__attribute__((warn_unused_result))
+#elif defined(__GNUC__)
+__attribute__((access(read_only, 1, 2), access(write_only, 3), warn_unused_result))
+#endif
+size_t libutf8_decode_mem(const char *text, size_t text_len, uint_least32_t *codepoint_out, unsigned flags);
+
+
+#if defined(__GNUC__)
+# pragma GCC diagnostic pop
+#endif
+
+#endif