diff options
39 files changed, 2507 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1bc81ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +*\#* +*~ +*.o +*.a +*.t +*.f +*.lo +*.to +*.fo +*.su +*.so +*.so.* +*.dll +*.dylib +*.gch +*.gcov +*.gcno +*.gcda diff --git a/LIBUTF8_ALLOW_2BYTE_NULL.3 b/LIBUTF8_ALLOW_2BYTE_NULL.3 new file mode 100644 index 0000000..51753c3 --- /dev/null +++ b/LIBUTF8_ALLOW_2BYTE_NULL.3 @@ -0,0 +1,26 @@ +.TH LIBUTF8_ALLOW_2BYTE_NULL 3 LIBUTF8 +.SH NAME +LIBUTF8_ALLOW_2BYTE_NULL - Accept the two-byte overlong encoding of U+0000 + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_ALLOW_2BYTE_NULL\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_ALLOW_2BYTE_NULL +macro is a flag for +.BR libutf8_decode (3) +and +.BR libutf8_decode_mem (3). +It causes those functions to accept the two-byte overlong +UTF-8 encoding of U+0000. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_ALLOW_LONG_ENCODING (3), +.BR libutf8_decode (3), +.BR libutf8_decode_mem (3) diff --git a/LIBUTF8_ALLOW_31_BITS.3 b/LIBUTF8_ALLOW_31_BITS.3 new file mode 100644 index 0000000..f86b59b --- /dev/null +++ b/LIBUTF8_ALLOW_31_BITS.3 @@ -0,0 +1,31 @@ +.TH LIBUTF8_ALLOW_31_BITS 3 LIBUTF8 +.SH NAME +LIBUTF8_ALLOW_31_BITS - Accept decoded codepoints up to 31 bits + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_ALLOW_31_BITS\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_ALLOW_31_BITS +macro is a flag for +.BR libutf8_decode (3) +and +.BR libutf8_decode_mem (3). +It causes those functions to accept codepoints greater than +.B LIBUTF8_ULTIMATE_CODEPOINT +when they fit in an unsigned 31-bit integer. +.PP +Such values are encoded with historical UTF-8 sequences of up to six bytes and +are not Unicode codepoints. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_ULTIMATE_CODEPOINT (3), +.BR libutf8_decode (3), +.BR libutf8_decode_mem (3), +.BR libutf8_encode (3) diff --git a/LIBUTF8_ALLOW_LONG_ENCODING.3 b/LIBUTF8_ALLOW_LONG_ENCODING.3 new file mode 100644 index 0000000..f17e80c --- /dev/null +++ b/LIBUTF8_ALLOW_LONG_ENCODING.3 @@ -0,0 +1,31 @@ +.TH LIBUTF8_ALLOW_LONG_ENCODING 3 LIBUTF8 +.SH NAME +LIBUTF8_ALLOW_LONG_ENCODING - Accept overlong UTF-8 encodings + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_ALLOW_LONG_ENCODING\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_ALLOW_LONG_ENCODING +macro is a flag for +.BR libutf8_decode (3) +and +.BR libutf8_decode_mem (3). +It causes those functions to accept overlong UTF-8 encodings +of up to six bytes, including the two-byte overlong encoding +of U+0000. +.PP +This flag includes the effect of +.BR LIBUTF8_ALLOW_2BYTE_NULL (3). +The two flags need not be combined. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_ALLOW_2BYTE_NULL (3), +.BR libutf8_decode (3), +.BR libutf8_decode_mem (3) diff --git a/LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT.3 b/LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT.3 new file mode 100644 index 0000000..4798685 --- /dev/null +++ b/LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT.3 @@ -0,0 +1,22 @@ +.TH LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT 3 LIBUTF8 +.SH NAME +LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT - Unicode replacement character as a codepoint + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT +macro expands the codepoint of U+FFFD REPLACEMENT CHARACTER +expressed as an unsigned integer that is at least 16 bits +wide and interpretable by the preprocessor + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_REPLACEMENT_CHARACTER_UTF8 (3), +.BR LIBUTF8_REPLACE_INVALID (3) diff --git a/LIBUTF8_REPLACEMENT_CHARACTER_UTF8.3 b/LIBUTF8_REPLACEMENT_CHARACTER_UTF8.3 new file mode 100644 index 0000000..c8ce4c6 --- /dev/null +++ b/LIBUTF8_REPLACEMENT_CHARACTER_UTF8.3 @@ -0,0 +1,21 @@ +.TH LIBUTF8_REPLACEMENT_CHARACTER_UTF8 3 LIBUTF8 +.SH NAME +LIBUTF8_REPLACEMENT_CHARACTER_UTF8 - Unicode replacement character encoded as UTF-8 + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_REPLACEMENT_CHARACTER_UTF8\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_REPLACEMENT_CHARACTER_UTF8 +macro expands to a string literal containing the UTF-8 encoding +of U+FFFD REPLACEMENT CHARACTER. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3), +.BR LIBUTF8_REPLACE_INVALID (3) diff --git a/LIBUTF8_REPLACE_INVALID.3 b/LIBUTF8_REPLACE_INVALID.3 new file mode 100644 index 0000000..3919226 --- /dev/null +++ b/LIBUTF8_REPLACE_INVALID.3 @@ -0,0 +1,37 @@ +.TH LIBUTF8_REPLACE_INVALID 3 LIBUTF8 +.SH NAME +LIBUTF8_REPLACE_INVALID - Replace decoded invalid Unicode codepoints + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_REPLACE_INVALID\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_REPLACE_INVALID +macro is a flag for +.BR libutf8_decode (3) +and +.BR libutf8_decode_mem (3). +It causes those functions to replace a decoded invalid Unicode codepoint with +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) +rather than reject it. +.PP +The flag applies to non-character codepoints (see +.BR libutf8_is_noncharacter (3) +for definition) and UTF-16 surrogates. +It does not make illegal byte sequences, overlong encodings, +or codepoints above +.B LIBUTF8_ULTIMATE_CODEPOINT +valid. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_RETURN_INVALID (3), +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3), +.BR libutf8_is_continuation (3), +.BR libutf8_decode (3), +.BR libutf8_decode_mem (3) diff --git a/LIBUTF8_RETURN_INVALID.3 b/LIBUTF8_RETURN_INVALID.3 new file mode 100644 index 0000000..61d8c0a --- /dev/null +++ b/LIBUTF8_RETURN_INVALID.3 @@ -0,0 +1,31 @@ +.TH LIBUTF8_RETURN_INVALID 3 LIBUTF8 +.SH NAME +LIBUTF8_RETURN_INVALID - Return decoded invalid Unicode codepoints as is + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_RETURN_INVALID\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_RETURN_INVALID +macro is a flag for +.BR libutf8_decode (3) +and +.BR libutf8_decode_mem (3). +It causes those functions to accept decoded non-character +codepoints and UTF-16 surrogates and store them unchanged. +.PP +The flag does not make illegal byte sequences, overlong +encodings, or codepoints above +.B LIBUTF8_ULTIMATE_CODEPOINT +valid. + +.SH SEE ALSO +.BR libutf8 (7), +.BR LIBUTF8_REPLACE_INVALID (3), +.BR libutf8_decode (3), +.BR libutf8_decode_mem (3) diff --git a/LIBUTF8_ULTIMATE_CODEPOINT.3 b/LIBUTF8_ULTIMATE_CODEPOINT.3 new file mode 100644 index 0000000..8c5e88c --- /dev/null +++ b/LIBUTF8_ULTIMATE_CODEPOINT.3 @@ -0,0 +1,28 @@ +.TH LIBUTF8_ULTIMATE_CODEPOINT 3 LIBUTF8 +.SH NAME +LIBUTF8_ULTIMATE_CODEPOINT - The greatest Unicode codepoint + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +#define \fBLIBUTF8_ULTIMATE_CODEPOINT\fP /* implementation omitted */ +.fi + +.SH DESCRIPTION +The +.B LIBUTF8_ULTIMATE_CODEPOINT +macro expands to 0x10FFFF, the greatest codepoint in the Unicode +codepoint-space, expressed as an unsigned integer that is at least +32 bits wide and interpretable by the preprocessor. +.PP +Use this value when a boundary for Unicode codepoints is needed. +Values greater than it are accepted by the decoding functions only with +.BR LIBUTF8_ALLOW_31_BITS (3). + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_encode (3), +.BR libutf8_is_invalid (3), +.BR libutf8_is_valid (3), +.BR LIBUTF8_ALLOW_31_BITS (3) @@ -0,0 +1,15 @@ +ISC License + +© 2026 Mattias Andrée <m@maandree.se> + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c2668df --- /dev/null +++ b/Makefile @@ -0,0 +1,145 @@ +.POSIX: + +CONFIGFILE = config.mk +include $(CONFIGFILE) + +OS = linux +# Linux: linux +# Mac OS: macos +# Windows: windows +include mk/$(OS).mk + + +LIB_MAJOR = 1 +LIB_MINOR = 0 +LIB_VERSION = $(LIB_MAJOR).$(LIB_MINOR) +LIB_NAME = utf8 + + +OBJ_PUBLIC_FUZZ =\ + libutf8_decode_mem.o + +OBJ_PUBLIC_DONT_FUZZ =\ + libutf8_is_noncharacter.o\ + libutf8_is_surrogate.o\ + libutf8_is_invalid.o\ + libutf8_is_valid.o\ + libutf8_is_continuation.o\ + libutf8_encode_measure.o\ + libutf8_encode.o\ + libutf8_decode.o + +OBJ =\ + $(OBJ_PUBLIC_FUZZ)\ + $(OBJ_PUBLIC_DONT_FUZZ) + +HDR =\ + libutf8.h\ + common.h + +MAN7 =\ + libutf8.7 + +MAN3 = $(OBJ:.o=.3)\ + LIBUTF8_ULTIMATE_CODEPOINT.3\ + LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT.3\ + LIBUTF8_REPLACEMENT_CHARACTER_UTF8.3\ + LIBUTF8_REPLACE_INVALID.3\ + LIBUTF8_RETURN_INVALID.3\ + LIBUTF8_ALLOW_31_BITS.3\ + LIBUTF8_ALLOW_2BYTE_NULL.3\ + LIBUTF8_ALLOW_LONG_ENCODING.3 + +LOBJ = $(OBJ:.o=.lo) +TOBJ = $(OBJ:.o=.to) +TEST = $(OBJ:.o=.t) +FOBJ = $(OBJ_PUBLIC_FUZZ:.o=.fo) +FUZZ = $(OBJ_PUBLIC_FUZZ:.o=.f) + + +ALL_CPPFLAGS = $(CPPFLAGS) $(FUZZED_CPPFLAGS) +ALL_CFLAGS = $(CFLAGS) $(FUZZED_CFLAGS) +ALL_LDFLAGS = $(LDFLAGS) $(FUZZED_LDFLAGS) + + +all: libutf8.a libutf8.$(LIBEXT) $(TEST) +$(OBJ): $(HDR) +$(LOBJ): $(HDR) +$(TOBJ): $(HDR) +$(FOBJ): $(HDR) +$(TEST): $(HDR) libutf8.a +$(FUZZ): $(HDR) libutf8.a + +.c.o: + $(CC) -c -o $@ $< $(ALL_CFLAGS) $(COV_CFLAGS) $(ALL_CPPFLAGS) $(COV_CPPFLAGS) + +.c.lo: + $(CC) -fPIC -c -o $@ $< $(ALL_CFLAGS) $(COV_CFLAGS) $(ALL_CPPFLAGS) $(COV_CPPFLAGS) + +.c.to: + $(CC) -DTEST -c -o $@ $< $(G) $(ALL_CFLAGS) $(ALL_CPPFLAGS) $(COV_CPPFLAGS) + +.to.t: + $(CC) -o $@ $< libutf8.a $(G) $(ALL_LDFLAGS) $(COV_LDFLAGS) + +.c.fo: + $(CC) -DTEST -DFUZZ -c -o $@ $< $(G) $(ALL_CFLAGS) $(ALL_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_CPPFLAGS) + +.fo.f: + $(CC) -o $@ $< libutf8.a $(G) $(ALL_LDFLAGS) $(TEST_LDFLAGS) $(FUZZ_LDFLAGS) + +libutf8.a: $(OBJ) + @rm -f -- $@ + $(AR) rc $@ $(OBJ) + $(AR) ts $@ > /dev/null + +libutf8.$(LIBEXT): $(LOBJ) + $(CC) $(LIBFLAGS) -o $@ $(LOBJ) $(ALL_LDFLAGS) + +check: $(TEST) + @set -ex;\ + for t in $(TEST); do\ + $(CHECK_PREFIX) ./$$t;\ + done +# Setting CHECK_PREFIX is intended for developers, setting it +# (specially to use valgrind(1)) may limit what the test code +# is able to test + +fuzz: $(FUZZ) + @set -ex;\ + for f in $(FUZZ); do\ + $(FUZZ_PREFIX) ./$$f $(FUZZ_SUFFIX);\ + done + +install: libutf8.a libutf8.$(LIBEXT) + mkdir -p -- "$(DESTDIR)$(PREFIX)/lib" + mkdir -p -- "$(DESTDIR)$(PREFIX)/include" + mkdir -p -- "$(DESTDIR)$(MANPREFIX)/man3" + mkdir -p -- "$(DESTDIR)$(MANPREFIX)/man7" + cp -- libutf8.a "$(DESTDIR)$(PREFIX)/lib/" + cp -- libutf8.$(LIBEXT) "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBMINOREXT)" + $(FIX_INSTALL_NAME) "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBMINOREXT)" + ln -sf -- libutf8.$(LIBMINOREXT) "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBMAJOREXT)" + ln -sf -- libutf8.$(LIBMAJOREXT) "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBEXT)" + cp -- libutf8.h "$(DESTDIR)$(PREFIX)/include/" + cp -P -- $(MAN3) "$(DESTDIR)$(MANPREFIX)/man3/" + cp -P -- $(MAN7) "$(DESTDIR)$(MANPREFIX)/man7/" + +uninstall: + -rm -f -- "$(DESTDIR)$(PREFIX)/lib/libutf8.a" + -rm -f -- "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBMAJOREXT)" + -rm -f -- "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBMINOREXT)" + -rm -f -- "$(DESTDIR)$(PREFIX)/lib/libutf8.$(LIBEXT)" + -rm -f -- "$(DESTDIR)$(PREFIX)/include/libutf8.h" + -cd -- "$(DESTDIR)$(MANPREFIX)/man3/" && rm -f -- $(MAN3) + -cd -- "$(DESTDIR)$(MANPREFIX)/man7/" && rm -f -- $(MAN7) + +clean: + -rm -f -- *.o *.a *.lo *.su *.so *.so.* *.dll *.dylib + -rm -f -- *.gch *.gcov *.gcno *.gcda *.$(LIBEXT) + -rm -f -- *.t *.to *.f *.fo + +.SUFFIXES: +.SUFFIXES: .lo .o .c .to .t .fo .f + +.PHONY: all check fuzz install uninstall clean @@ -0,0 +1,47 @@ +NAME + libutf8 - UTF-8 encoding and decoding + +SYNOPSIS + #include <libutf8.h> + + Link with -lutf8. + +DESCRIPTION + The libutf8 library provides functions for encoding and decoding + UTF-8 sequences, with flexibility in how invalid input is handled. + + The library implements the following functions: + + Encoding functions: + libutf8_encode(3) + Encode a codepoint as UTF-8 + + libutf8_encode_measure(3) + Get the number of bytes required to encode a codepoint + + Decoding functions: + libutf8_decode(3) + Decode one sequence from a string + + libutf8_decode_mem(3) + Decode one sequence from a memory buffer + + Codepoint and byte test functions: + libutf8_is_noncharacter(3) + Test whether a codepoint is a Unicode non-character + codepoint + + libutf8_is_surrogate(3) + Test whether a codepoint is a UTF-16 surrogate + + libutf8_is_invalid(3) + Test whether a codepoint is invalid Unicode + + libutf8_is_valid(3) + Test whether a codepoint is valid Unicode + + libutf8_is_continuation(3) + Test whether a byte is a UTF-8 continuation byte + +SEE ALSO + libgrapheme(7) diff --git a/common.h b/common.h new file mode 100644 index 0000000..3faa5bf --- /dev/null +++ b/common.h @@ -0,0 +1,163 @@ +/* See LICENSE file for copyright and license details. */ +#include "libutf8.h" + + +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wunsafe-buffer-usage" +# pragma clang diagnostic ignored "-Wimplicit-fallthrough" +#endif + +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Winline" +#endif + + +#define DECODE(LENGTH_CHECK)\ + do {\ + static const uint_least32_t limits[] = {\ + (UINT32_C(1) << 0),\ + (UINT32_C(1) << 7),\ + (UINT32_C(1) << 11),\ + (UINT32_C(1) << 16),\ + (UINT32_C(1) << 21),\ + (UINT32_C(1) << 26),\ + (UINT32_C(1) << 31)\ + };\ + \ + uint_least32_t codepoint, lower_limit;\ + unsigned char c;\ + size_t n, char_len = 0u;\ + \ + c = ((const unsigned char *)text)[char_len++];\ + \ + n = 0u;\ + while (c & 0x80u) {\ + c = (unsigned char)(c << 1);\ + n += 1u;\ + }\ + \ + if (n == 0u) {\ + codepoint = (uint_least32_t)c;\ + goto out;\ + }\ + if (n == 1u || n >= sizeof(limits) / sizeof(*limits))\ + return 0u;\ + \ + LENGTH_CHECK\ + \ + lower_limit = limits[n - 1u];\ + \ + codepoint = (uint_least32_t)((c & 0xFFu) >> n);\ + while (--n) {\ + c = ((const unsigned char *)text)[char_len++];\ + if ((c & 0xC0u) != 0x80u)\ + return 0u;\ + codepoint <<= 6;\ + codepoint |= (uint_least32_t)(c & 0x3Fu);\ + }\ + \ + if (codepoint < lower_limit && !(flags & LIBUTF8_ALLOW_LONG_ENCODING))\ + if (codepoint || char_len > 2u || !(flags & LIBUTF8_ALLOW_2BYTE_NULL))\ + return 0u;\ + \ + if (codepoint > LIBUTF8_ULTIMATE_CODEPOINT) {\ + if (!(flags & LIBUTF8_ALLOW_31_BITS) || codepoint >> 31)\ + return 0u;\ + } else if (flags & LIBUTF8_REPLACE_INVALID) {\ + if (libutf8_is_noncharacter(codepoint) || libutf8_is_surrogate(codepoint))\ + codepoint = LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT;\ + } else if (!(flags & LIBUTF8_RETURN_INVALID)) {\ + if (libutf8_is_noncharacter(codepoint) || libutf8_is_surrogate(codepoint))\ + return 0u;\ + }\ + \ + out:\ + if (codepoint_out)\ + *codepoint_out = codepoint;\ + return char_len;\ + } while (0) + + +#define ENCODE(N)\ + do {\ + switch (N) {\ + default:\ + break;\ + case 1u:\ + out_buffer[0u] = (char)codepoint;\ + break;\ + case 6u:\ + out_buffer[5u] = (char)(0x80u | ((unsigned char)(codepoint & 0x3Fu)));\ + codepoint >>= 6;\ + /* fall through */\ + case 5u:\ + out_buffer[4u] = (char)(0x80u | ((unsigned char)(codepoint & 0x3Fu)));\ + codepoint >>= 6;\ + /* fall through */\ + case 4u:\ + out_buffer[3u] = (char)(0x80u | ((unsigned char)(codepoint & 0x3Fu)));\ + codepoint >>= 6;\ + /* fall through */\ + case 3u:\ + out_buffer[2u] = (char)(0x80u | ((unsigned char)(codepoint & 0x3Fu)));\ + codepoint >>= 6;\ + /* fall through */\ + case 2u:\ + out_buffer[1u] = (char)(0x80u | ((unsigned char)(codepoint & 0x3Fu)));\ + codepoint >>= 6;\ + \ + out_buffer[0u] = (char)(((0xFFu << (8u - (N))) & 0xFFu) | codepoint);\ + break;\ + }\ + } while (0) + + +#ifdef TEST +# ifdef __linux__ +# include <sys/prctl.h> +# endif +# include <sys/resource.h> +# include <assert.h> +# include <stdio.h> +# include <stdlib.h> +# include <string.h> +# include <time.h> +# include <unistd.h> + +# define ARM_ALARM()\ + do {\ + unsigned int alarm_time__ = alarm(10u);\ + if (alarm_time__ > 10u)\ + alarm(alarm_time__);\ + } while (0) + +# if defined(PR_SET_DUMPABLE) +# define DISABLE_COREDUMP()\ + do {\ + struct rlimit rl__;\ + rl__.rlim_cur = 0;\ + rl__.rlim_max = 0;\ + (void) setrlimit(RLIMIT_CORE, &rl__);\ + (void) prctl(PR_SET_DUMPABLE, 0);\ + } while (0) +# else +# define DISABLE_COREDUMP()\ + do {\ + struct rlimit rl__;\ + rl__.rlim_cur = 0;\ + rl__.rlim_max = 0;\ + (void) setrlimit(RLIMIT_CORE, &rl__);\ + } while (0) +# endif + +# define SEED_RAND()\ + srand((unsigned)time(NULL)) + +# define SET_UP_TEST()\ + do {\ + ARM_ALARM();\ + DISABLE_COREDUMP();\ + SEED_RAND();\ + } while (0) + +#endif diff --git a/config-coverage-gcc.mk b/config-coverage-gcc.mk new file mode 100644 index 0000000..60f81f8 --- /dev/null +++ b/config-coverage-gcc.mk @@ -0,0 +1,14 @@ +CONFIGFILE_PROPER = config.mk +include $(CONFIGFILE_PROPER) + +CC = $(CC_PREFIX)gcc -std=c99 +GCOV = gcov + +COV_CPPFLAGS = -DCOVERAGE_TEST +COV_CFLAGS = --coverage -g -O0 +COV_LDFLAGS = --coverage -g -O0 + +G = + +coverage: check + $(GCOV) -pr -- *.gcda 2>&1 diff --git a/config-fuzz-clang.mk b/config-fuzz-clang.mk new file mode 100644 index 0000000..d7c8cb3 --- /dev/null +++ b/config-fuzz-clang.mk @@ -0,0 +1,11 @@ +CONFIGFILE_PROPER = config.mk +include $(CONFIGFILE_PROPER) + +CC = $(CC_PREFIX)clang -std=c99 + +SANITIZE = $(CLANG_SANITIZE) +FUZZ_CFLAGS = -fsanitize=fuzzer +FUZZ_LDFLAGS = -fsanitize=fuzzer +FUZZED_CPPFLAGS = -DFUZZ + +fuzz: diff --git a/config.mk b/config.mk new file mode 100644 index 0000000..0d98cb2 --- /dev/null +++ b/config.mk @@ -0,0 +1,16 @@ +PREFIX = /usr +MANPREFIX = $(PREFIX)/share/man + +CC = c99 + +COMMON_SANITIZE = -fsanitize=alignment,shift,signed-integer-overflow,object-size,null,undefined,bounds,address +CLANG_SANITIZE = -O1 $(COMMON_SANITIZE),cfi -flto -fvisibility=hidden -fno-sanitize-trap=cfi +GCC_SANITIZE = -O1 $(COMMON_SANITIZE) +#SANITIZE = $(CLANG_SANITIZE) +#SANITIZE = $(GCC_SANITIZE) + +CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 -D_GNU_SOURCE +CFLAGS = $(SANITIZE) +LDFLAGS = $(SANITIZE) + +G = -g diff --git a/libutf8.7 b/libutf8.7 new file mode 100644 index 0000000..153fcdf --- /dev/null +++ b/libutf8.7 @@ -0,0 +1,83 @@ +.TH LIBUTF8 7 LIBUTF8 +.SH NAME +libutf8 - UTF-8 encoding and decoding + +.SH SYNOPSIS +.ni +#include <libutf8.h> +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.B libutf8 +library provides functions for encoding and decoding +UTF-8 sequences, with flexibility in how invalid input is handled. +.PP +The library implements the following functions and macros: +.sp +.SS Encoding functions: +.TP +.BR libutf8_encode (3) +Encode a codepoint as UTF-8 +.TP +.BR libutf8_encode_measure (3) +Get the number of bytes required to encode a codepoint +.sp +.SS Decoding functions: +.TP +.BR libutf8_decode (3) +Decode one sequence from a string +.TP +.BR libutf8_decode_mem (3) +Decode one sequence from a memory buffer +.sp +.SS Codepoint and byte test functions: +.TP +.BR libutf8_is_noncharacter (3) +Test whether a codepoint is a Unicode non-character codepoint +.TP +.BR libutf8_is_surrogate (3) +Test whether a codepoint is a UTF-16 surrogate +.TP +.BR libutf8_is_invalid (3) +Test whether a codepoint is invalid Unicode +.TP +.BR libutf8_is_valid (3) +Test whether a codepoint is valid Unicode +.TP +.BR libutf8_is_continuation (3) +Test whether a byte is a UTF-8 continuation byte +.sp +.SS Macro constants: +.TP +.BR LIBUTF8_ULTIMATE_CODEPOINT (3) +The greatest Unicode codepoint +.TP +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) +The Unicode replacement character as a codepoint +.TP +.BR LIBUTF8_REPLACEMENT_CHARACTER_UTF8 (3) +The Unicode replacement character encoded as UTF-8 +.sp +.SS Decoding flags: +.TP +.BR LIBUTF8_REPLACE_INVALID (3) +Replace decoded invalid Unicode codepoints +.TP +.BR LIBUTF8_RETURN_INVALID (3) +Return decoded invalid Unicode codepoints as is +.TP +.BR LIBUTF8_ALLOW_31_BITS (3) +Accept decoded codepoints up to 31 bits +.TP +.BR LIBUTF8_ALLOW_2BYTE_NULL (3) +Accept the two-byte overlong encoding of U+0000 +.TP +.BR LIBUTF8_ALLOW_LONG_ENCODING (3) +Accept overlong UTF-8 encodings + +.SH SEE ALSO +.BR libgrapheme (7) diff --git a/libutf8.h b/libutf8.h new file mode 100644 index 0000000..8feeebf --- /dev/null +++ b/libutf8.h @@ -0,0 +1,351 @@ +/* See LICENSE file for copyright and license details. */ +#ifndef LIBUTF8_H +#define LIBUTF8_H + +#include <stddef.h> +#include <stdint.h> + +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Winline" +#endif + + +/** + * The greatest Unicode codepoint + * + * @seealso libutf8_is_invalid + */ +#define LIBUTF8_ULTIMATE_CODEPOINT UINT32_C(0x10FFFF) + + +/** + * The Unicode replacement character as a UTF-32 codepoint + * + * @seealso LIBUTF8_REPLACEMENT_CHARACTER_UTF8 + */ +#define LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT 0xFFFDu + + +/** + * The Unicode replacement character encoded as UTF-8 + * + * @seealso LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT + */ +#define LIBUTF8_REPLACEMENT_CHARACTER_UTF8 "\xEF\xBF\xBD" + + +/** + * Replace a decoded invalid codepoint with + * `LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT` + * + * @seealso LIBUTF8_RETURN_INVALID + */ +#define LIBUTF8_REPLACE_INVALID 0x0001u + + +/** + * Return a decoded invalid codepoint unchanged + * + * @seealso LIBUTF8_REPLACE_INVALID + */ +#define LIBUTF8_RETURN_INVALID 0x0002u + + +/** + * Accept UTF-8 sequences that decode past + * `LIBUTF8_ULTIMATE_CODEPOINT` to a maximum of 31 bits + */ +#define LIBUTF8_ALLOW_31_BITS 0x0004u + + +/** + * Accept the two-byte overlong encoding of U+0000 + * + * @seealso LIBUTF8_ALLOW_LONG_ENCODING + */ +#define LIBUTF8_ALLOW_2BYTE_NULL 0x0008u + + +/** + * Accept all overlong UTF-8 encodings + * + * This flag implies LIBUTF8_ALLOW_2BYTE_NULL + * + * @seealso LIBUTF8_ALLOW_2BYTE_NULL + */ +#define LIBUTF8_ALLOW_LONG_ENCODING 0x0010u + + +/** + * Test whether a codepoint is a non-character Unicode codepoint + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is a non-character Unicode codepoint, 0 otherwise + * + * A non-character Unicode codepoint is codepoint + * reserved as invalid and without special meaning + * + * The result is unspecified if `codepoint` is + * greater than LIBUTF8_ULTIMATE_CODEPOINT + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_noncharacter(uint_least32_t codepoint) +{ + return (0xFDD0u <= codepoint && codepoint <= 0xFDEFu) || (codepoint & 0xFFFEu) == 0xFFFEu; +} + + +/** + * Test whether a codepoint is a UTF-16 surrogate + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is a UTF-16 surrogate, 0 otherwise + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_surrogate(uint_least32_t codepoint) +{ + return 0xD800u <= codepoint && codepoint <= 0xDFFFu; +} + + +/** + * Test whether a codepoint is invalid Unicode + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is outside the Unicode range, + * a surrogate, or a non-character; 0 otherwise + * + * @seealso libutf8_is_valid + * @seealso LIBUTF8_ULTIMATE_CODEPOINT + * @seealso libutf8_is_noncharacter + * @seealso libutf8_is_surrogate + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_invalid(uint_least32_t codepoint) +{ + return codepoint > LIBUTF8_ULTIMATE_CODEPOINT || + libutf8_is_noncharacter(codepoint) || + libutf8_is_surrogate(codepoint); +} + + +/** + * Test whether a codepoint is valid Unicode + * + * @param codepoint The codepoint to test + * @return 1 if `codepoint` is within the Unicode range + * and is neither a surrogate nor a non-character, + * 0 otherwise + * + * @seealso libutf8_is_invalid + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_valid(uint_least32_t codepoint) +{ + return !libutf8_is_invalid(codepoint); +} + + +/** + * Test whether a byte is a UTF-8 continuation byte + * + * @param c The byte to test + * @return 1 if `c` is a codepoint continuation byte, + * 0 otherwise + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline int +libutf8_is_continuation(char c) +{ + return ((unsigned char)c & 0xC0u) == 0x80u; +} + + +/** + * Get the number of bytes required to encode a codepoint + * + * @param codepoint The codepoint to measure + * @return The number of bytes required to encode `codepoint`; + * 0 if `codepoint` does not fit in an unsigned 31-bit integer + * + * The maximum return value is 4 for valid Unicode + * codepoints and 6 for any value accept by `libutf8_encode` + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EDOM` when this + * function returns 0 + * + * @seealso libutf8_encode + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__GNUC__) +__attribute__((const, warn_unused_result)) +#endif +inline size_t +libutf8_encode_measure(uint_least32_t codepoint) +{ + if (codepoint >> 7 == 0u) return 1u; + if (codepoint >> 11 == 0u) return 2u; + if (codepoint >> 16 == 0u) return 3u; + if (codepoint >> 21 == 0u) return 4u; + if (codepoint >> 26 == 0u) return 5u; + if (codepoint >> 31 == 0u) return 6u; + return 0u; +} + + +/** + * Encode a codepoint as UTF-8 + * + * @param out_buffer Output buffer; must have room for the number + * of bytes required to encode `codepoint` + * @param codepoint The codepoint to encode + * @return The number of bytes written + * + * NB! The output is not NUL-terminated + * + * Besides Unicode codepoints, this function encodes 31-bit + * values greater than `LIBUTF8_ULTIMATE_CODEPOINT` using + * historical UTF-8 sequences up to 6 bytes. This lets + * applications embed application-specific data in a file. + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EDOM` when this + * function returns 0 + * + * @seealso libutf8_encode_measure + * @seealso libutf8_is_valid + * @seealso libutf8_decode + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((nonnull(1), warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((nonnull(1), access(write_only, 1), warn_unused_result)) +#endif +size_t libutf8_encode(char *out_buffer, uint_least32_t codepoint); + + +/** + * Decode the first UTF-8 sequence in a string + * + * @param text The input string; must contain a complete candidate sequence + * @param codepoint_out Unless `NULL`, output for the decoded codepoint + * @param flags Bitwise OR of zero or more or the following flags: + * - LIBUTF8_REPLACE_INVALID: + * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, + * with the REPLACEMENT CHARACTER. + * NB! This only affects invalid codepoints, not illegal byte sequences. + * - LIBUTF8_RETURN_INVALID: + * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid + * - LIBUTF8_ALLOW_31_BITS + * Treat 31-bit codepoints greater than + * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. + * - LIBUTF8_ALLOW_2BYTE_NULL + * Accept the overlong 2-byte encoding of U+0000. + * - LIBUTF8_ALLOW_LONG_ENCODING + * Accept overlong encodings, up to 6 bytes. + * @return The number of decoded bytes, 0 if the input is invalid + * + * It is unspecified which flag dominates: + * LIBUTF8_REPLACE_INVALID or LIBUTF8_RETURN_INVALID + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EILSEQ` when this + * function returns 0 + * + * @seealso libutf8_decode_mem + * @seealso libutf8_is_continuation + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((nonnull(1), warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((nonnull(1), access(write_only, 2), warn_unused_result)) +#endif +size_t libutf8_decode(const char *text, uint_least32_t *codepoint_out, unsigned flags); + + +/** + * Decode the first UTF-8 sequence in a string + * + * @param text The input string; must contain a complete candidate sequence + * @param text_len The number of bytes available in `text` + * @param codepoint_out Unless `NULL`, output for the decoded codepoint + * @param flags Bitwise OR of zero or more or the following flags: + * - LIBUTF8_REPLACE_INVALID: + * Replace invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, + * with the REPLACEMENT CHARACTER. + * NB! This only affects invalid codepoints, not illegal byte sequences. + * - LIBUTF8_RETURN_INVALID: + * Treat invalid codepoints, up to `LIBUTF8_ULTIMATE_CODEPOINT`, as valid + * - LIBUTF8_ALLOW_31_BITS + * Treat 31-bit codepoints greater than + * `LIBUTF8_ULTIMATE_CODEPOINT` as valid. + * - LIBUTF8_ALLOW_2BYTE_NULL + * Accept the overlong 2-byte encoding of U+0000. + * - LIBUTF8_ALLOW_LONG_ENCODING + * Accept overlong encodings, up to 6 bytes. + * @return The number of decoded bytes, 0 if the input is invalid; + * or the required sequence length specified by a valid + * leading byte if `text_len` is less than this value + * (input is truncated) + * + * Note, that a return value greater than `text_len` does not + * necessarily (it is unspecified) indicate that all available + * bytes are invalid, only that the first byte is valid + * + * This function never sets `errno`, however an + * appropriate `errno`-value is `EILSEQ` when this + * function returns 0 + * + * @seealso libutf8_decode + * @seealso libutf8_is_continuation + * + * This function is MT-Safe, AS-Safe, and AC-Safe + */ +#if defined(__clang__) +__attribute__((warn_unused_result)) +#elif defined(__GNUC__) +__attribute__((access(read_only, 1, 2), access(write_only, 3), warn_unused_result)) +#endif +size_t libutf8_decode_mem(const char *text, size_t text_len, uint_least32_t *codepoint_out, unsigned flags); + + +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif + +#endif diff --git a/libutf8_decode.3 b/libutf8_decode.3 new file mode 100644 index 0000000..fd97cb2 --- /dev/null +++ b/libutf8_decode.3 @@ -0,0 +1,114 @@ +.TH LIBUTF8_DECODE 3 LIBUTF8 +.SH NAME +libutf8_decode - Decode one UTF-8 sequence from a string + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +size_t \fBlibutf8_decode\fP(const char *\fItext\fP, uint_least32_t *\fIcodepoint_out\fP, unsigned \fIflags\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_decode () +function decodes the first UTF-8 sequence in +.IR text . + +Unless the +.BR libutf8_decode () +function returns 0, and unless +.I codepoint_out +is +.IR NULL , +the decoded codepoint is stored in +.IR *codepoint_out . +.PP +.I flags +is a bitwise OR of zero or more of the following values: +.TP +.B LIBUTF8_REPLACE_INVALID +Replace decoded invalid codepoints up to +.B LIBUTF8_ULTIMATE_CODEPOINT +with the Unicode REPLACEMENT CHARACTER (U+FFFD). +This does not affect illegal byte sequences. +.TP +.B LIBUTF8_RETURN_INVALID +Return decoded invalid codepoints up to +.B LIBUTF8_ULTIMATE_CODEPOINT +unchanged. +.TP +.B LIBUTF8_ALLOW_31_BITS +Accept codepoints greater than +.BR LIBUTF8_ULTIMATE_CODEPOINT , +up to representable with 31 bits, unsigned. +.TP +.B LIBUTF8_ALLOW_2BYTE_NULL +Accept the two-byte overlong encoding of U+0000. +.TP +.B LIBUTF8_ALLOW_LONG_ENCODING +Accept overlong encodings up to six bytes. +This includes the encoding accepted by +.BR LIBUTF8_ALLOW_2BYTE_NULL . +.PP +It is unspecified which flag dominates if both +.B LIBUTF8_REPLACE_INVALID +and +.B LIBUTF8_RETURN_INVALID +are set. + +.SH RETURN VALUE +The +.BR libutf8_decode () +function returns the number of decoded bytes, +or zero if the input is invalid. + +.SH ERRORS +The function cannot fail per se, and never sets +.IR errno . +.B EILSEQ +is an appropriate value when it returns zero. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_decode () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH APPLICATION USAGE +Application that use +.BR LIBUTF8_REPLACE_INVALID (3) +will probably also want to skip the first +undecoded by when 0 is returned and insert +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) +in its place, and optionally skip the following +bytes until there is one for which +.BR libutf8_is_continuation (3) +returns zero. + +.SH NOTES +The +.BR libutf8_decode () +function can read up +to six bytes while examining a malformed sequence; use +.BR libutf8_decode_mem (3) +for a length-bounded buffer. + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_decode_mem (3), +.BR libutf8_is_continuation (3), +.BR libutf8_is_surrogate (3), +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) diff --git a/libutf8_decode.c b/libutf8_decode.c new file mode 100644 index 0000000..c821d0a --- /dev/null +++ b/libutf8_decode.c @@ -0,0 +1,213 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +size_t +libutf8_decode(const char *text, uint_least32_t *codepoint_out, unsigned flags) +{ + DECODE(); +} + + +#else + + +# define NONE (uint_least32_t)~(uint_least32_t)0 + + +static int +long_encode(char *out_buffer, uint_least32_t codepoint, size_t n) +{ + if (codepoint >> 31) { + if (n != 6u) + return 0; + } else { + if (libutf8_encode_measure(codepoint) > n || n > 6u) + return 0; + } + ENCODE(n); + return 1; +} + + +static void +check(const char *buf, uint_least32_t codepoint, size_t encode_len, unsigned flags, int overlong) +{ + uint_least32_t expect_codepoint, replacement, res; + size_t decode_len; + int extraultimate; + + extraultimate = (codepoint > (uint_least32_t)LIBUTF8_ULTIMATE_CODEPOINT && !(codepoint >> 31)); + + expect_codepoint = codepoint; + replacement = (uint_least32_t)LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT; + + res = NONE; + decode_len = libutf8_decode(buf, &res, flags | LIBUTF8_RETURN_INVALID); + if (overlong && + !(flags & LIBUTF8_ALLOW_LONG_ENCODING) && + !(encode_len == 2u && !codepoint && (flags & LIBUTF8_ALLOW_2BYTE_NULL)) && + !decode_len) + goto expect_zero; + if (extraultimate && !decode_len) + goto expect_zero; + if (codepoint >> 31) + goto expect_zero; + assert(decode_len == encode_len); + assert(res == expect_codepoint); + + assert(libutf8_decode(buf, NULL, flags | LIBUTF8_RETURN_INVALID) == encode_len); + + if (extraultimate && (flags & LIBUTF8_ALLOW_31_BITS)) + goto normal; + + if (libutf8_is_invalid(codepoint)) { + res = NONE; + assert(libutf8_decode(buf, &res, 0u) == 0u); + assert(res == NONE); + + res = NONE; + assert(libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID) == encode_len); + assert(res == replacement); + + res = NONE; + decode_len = libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID | LIBUTF8_RETURN_INVALID); + assert(!decode_len || decode_len == encode_len); + if (decode_len) + assert(res == replacement); + else + assert(res == NONE); + } else { + normal: + res = NONE; + assert(libutf8_decode(buf, &res, flags) == encode_len); + assert(res == expect_codepoint); + + res = NONE; + assert(libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID) == encode_len); + assert(res == expect_codepoint); + + res = NONE; + assert(libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID | LIBUTF8_RETURN_INVALID) == encode_len); + assert(res == expect_codepoint); + } + + return; + +expect_zero: + assert(!libutf8_decode(buf, &res, flags)); + assert(!libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID)); + assert(!libutf8_decode(buf, &res, flags | LIBUTF8_REPLACE_INVALID | LIBUTF8_RETURN_INVALID)); +} + + +static void +check_codepoint(uint_least32_t codepoint) +{ + static const unsigned extra_flags[] = { + 0u, + LIBUTF8_ALLOW_31_BITS, + LIBUTF8_ALLOW_2BYTE_NULL, + LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_2BYTE_NULL, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_2BYTE_NULL | LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_2BYTE_NULL | LIBUTF8_ALLOW_LONG_ENCODING + }; + + static size_t corruption = 0u; + + size_t flags_i, encode_len; + char buf[128]; + int overlong; + unsigned flags; + size_t q, r; + + for (flags_i = 0u; flags_i < sizeof(extra_flags) / sizeof(*extra_flags); flags_i++) { + flags = extra_flags[flags_i]; + + overlong = 0; + for (encode_len = 1u; encode_len <= 6u; encode_len++) { + if (!long_encode(buf, codepoint, encode_len)) + continue; + + check(buf, codepoint, encode_len, flags, overlong); + + overlong = 1; + } + } + + if (codepoint >> 7) { + encode_len = libutf8_encode_measure(codepoint); + encode_len = encode_len ? encode_len : 6u; + long_encode(buf, codepoint, encode_len); + q = corruption / 3u; + r = corruption % 3u; + q %= encode_len - 1u; + buf[q + 1u] = (char)((unsigned char)buf[q + 1u] ^ ((r ^ 3u) << 6)); + corruption += 1u; + assert(libutf8_decode(buf, &(uint_least32_t){0u}, ~0u) == 0u); + } +} + + +int +main(void) +{ + static const char *rejected[] = { + "\x80\x80\x80\x80\x80\x80\x80\x80", + "\xFE\x80\x80\x80\x80\x80\x80\x80", + "\xFF\x80\x80\x80\x80\x80\x80\x80", + NULL + }; + + uint_least32_t cp, res; + size_t i; + + SET_UP_TEST(); + + for (cp = 0u; cp <= (uint_least32_t)LIBUTF8_ULTIMATE_CODEPOINT;) { + check_codepoint(cp); + if (!(++cp & 0x7FFFu)) + ARM_ALARM(); + } + + check_codepoint((uint_least32_t)0x110000ul); + check_codepoint((uint_least32_t)(1ul << 22)); + check_codepoint((uint_least32_t)(1ul << 23)); + check_codepoint((uint_least32_t)(1ul << 24)); + check_codepoint((uint_least32_t)(1ul << 25)); + check_codepoint((uint_least32_t)(1ul << 26)); + check_codepoint((uint_least32_t)(1ul << 27)); + check_codepoint((uint_least32_t)(1ul << 28)); + check_codepoint((uint_least32_t)(1ul << 29)); + check_codepoint((uint_least32_t)(1ul << 30)); + + check_codepoint((uint_least32_t)0x11FFFFul); + check_codepoint((uint_least32_t)((1ul << 22) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 23) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 24) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 25) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 26) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 27) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 28) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 29) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 30) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 31) - 1ul)); + + check_codepoint((uint_least32_t)(1ul << 31)); + check_codepoint((uint_least32_t)0xFFFFFFFFul); + + for (i = 0u; rejected[i]; i++) { + res = NONE; + assert(libutf8_decode(rejected[i], &res, ~0u) == 0u); + assert(res == NONE); + assert(libutf8_decode(rejected[i], NULL, ~0u) == 0u); + } + + return 0; +} + + +#endif diff --git a/libutf8_decode_mem.3 b/libutf8_decode_mem.3 new file mode 100644 index 0000000..aa66cba --- /dev/null +++ b/libutf8_decode_mem.3 @@ -0,0 +1,124 @@ +.TH LIBUTF8_DECODE_MEM 3 LIBUTF8 +.SH NAME +libutf8_decode_mem - Decode one UTF-8 sequence from a memory buffer + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +size_t \fBlibutf8_decode_mem\fP(const char *\fItext\fP, size_t \fItext_len\fP, uint_least32_t *\fIcodepoint_out\fP, unsigned \fIflags\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_decode_mem () +function decodes the first UTF-8 sequence in +.IR text , +however it the sequence is longer than +.I text_len +it reads the first byte and returns the +sequence length indicated by that byte. In +this case it is unspecified whether it reads +any other byte than the first byte. + +Unless the +.BR libutf8_decode_mem () +function returns 0 or a value greater than +.IR text_len , +and unless +.I codepoint_out +is +.IR NULL , +the decoded codepoint is stored in +.IR *codepoint_out . +.PP +.I flags +is a bitwise OR of zero or more of the following values: +.TP +.B LIBUTF8_REPLACE_INVALID +Replace decoded invalid codepoints up to +.B LIBUTF8_ULTIMATE_CODEPOINT +with the Unicode REPLACEMENT CHARACTER (U+FFFD). +This does not affect illegal byte sequences. +.TP +.B LIBUTF8_RETURN_INVALID +Return decoded invalid codepoints up to +.B LIBUTF8_ULTIMATE_CODEPOINT +unchanged. +.TP +.B LIBUTF8_ALLOW_31_BITS +Accept codepoints greater than +.BR LIBUTF8_ULTIMATE_CODEPOINT , +up to representable with 31 bits, unsigned. +.TP +.B LIBUTF8_ALLOW_2BYTE_NULL +Accept the two-byte overlong encoding of U+0000. +.TP +.B LIBUTF8_ALLOW_LONG_ENCODING +Accept overlong encodings up to six bytes. +This includes the encoding accepted by +.BR LIBUTF8_ALLOW_2BYTE_NULL . +.PP +It is unspecified which flag dominates if both +.B LIBUTF8_REPLACE_INVALID +and +.B LIBUTF8_RETURN_INVALID +are set. + +.SH RETURN VALUE +The +.BR libutf8_decode_mem () +function returns the number of decoded bytes, +or zero if the input is invalid. However if +the sequence is longer than +.I text_len +is returns the indicated sequence length. +.PP +It is unspecified which is returned, 0 or +the indicated sequence length when the leading +byte indicates a length longer than +.I text_len +but the other available bytes do not create +a valid continuation. + +.SH ERRORS +The function cannot fail per se, and never sets +.IR errno . +.B EILSEQ +is an appropriate value when it returns zero. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_decode_mem () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH APPLICATION USAGE +Application that use +.BR LIBUTF8_REPLACE_INVALID (3) +will probably also want to skip the first +undecoded by when 0 is returned and insert +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) +in its place, and optionally skip the following +bytes until there is one for which +.BR libutf8_is_continuation (3) +returns zero. + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_decode (3), +.BR libutf8_is_continuation (3), +.BR libutf8_is_surrogate (3), +.BR LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT (3) diff --git a/libutf8_decode_mem.c b/libutf8_decode_mem.c new file mode 100644 index 0000000..5d3b9ea --- /dev/null +++ b/libutf8_decode_mem.c @@ -0,0 +1,279 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +size_t +libutf8_decode_mem(const char *text, size_t text_len, uint_least32_t *codepoint_out, unsigned flags) +{ + if (!text_len) + return 0u; + + DECODE(if (n > text_len) return n;); +} + + +#else +# ifndef FUZZ + + +# define NONE (uint_least32_t)~(uint_least32_t)0 + + +static int +long_encode(char *out_buffer, uint_least32_t codepoint, size_t n) +{ + if (codepoint >> 31) { + if (n != 6u) + return 0; + } else { + if (libutf8_encode_measure(codepoint) > n || n > 6u) + return 0; + } + ENCODE(n); + return 1; +} + + +static void +check(const char *buf, uint_least32_t codepoint, size_t decode_avail, size_t encode_len, unsigned flags, int overlong) +{ + uint_least32_t expect_codepoint, replacement, res; + size_t decode_len; + int extraultimate; + + extraultimate = (codepoint > (uint_least32_t)LIBUTF8_ULTIMATE_CODEPOINT && !(codepoint >> 31)); + + if (decode_avail >= encode_len) { + expect_codepoint = codepoint; + replacement = (uint_least32_t)LIBUTF8_REPLACEMENT_CHARACTER_CODEPOINT; + } else { + expect_codepoint = NONE; + replacement = expect_codepoint; + } + + res = NONE; + decode_len = libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_RETURN_INVALID); + if (overlong && + !(flags & LIBUTF8_ALLOW_LONG_ENCODING) && + !(encode_len == 2u && !codepoint && (flags & LIBUTF8_ALLOW_2BYTE_NULL)) && + !decode_len) + goto expect_zero; + if (extraultimate && !decode_len) + goto expect_zero; + if (codepoint >> 31) + goto expect_zero; + assert(decode_len == encode_len); + assert(res == expect_codepoint); + + assert(libutf8_decode_mem(buf, decode_avail, NULL, flags | LIBUTF8_RETURN_INVALID) == encode_len); + + if (extraultimate && (flags & LIBUTF8_ALLOW_31_BITS)) + goto normal; + + if (libutf8_is_invalid(codepoint)) { + decode_len = libutf8_decode_mem(buf, decode_avail, &res, 0u); + if ((decode_len && decode_avail < encode_len) || decode_avail < 2u) + goto normal; + + res = NONE; + assert(decode_len == 0u); + assert(res == NONE); + + res = NONE; + assert(libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID) == encode_len); + assert(res == replacement); + + res = NONE; + decode_len = libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID + | LIBUTF8_RETURN_INVALID); + assert(!decode_len || decode_len == encode_len); + if (decode_len) + assert(res == replacement); + else + assert(res == NONE); + } else { + normal: + res = NONE; + assert(libutf8_decode_mem(buf, decode_avail, &res, flags) == encode_len); + assert(res == expect_codepoint); + + res = NONE; + assert(libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID) == encode_len); + assert(res == expect_codepoint); + + res = NONE; + assert(libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID + | LIBUTF8_RETURN_INVALID) == encode_len); + assert(res == expect_codepoint); + } + + return; + +expect_zero: + assert(!libutf8_decode_mem(buf, decode_avail, &res, flags)); + assert(!libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID)); + assert(!libutf8_decode_mem(buf, decode_avail, &res, flags | LIBUTF8_REPLACE_INVALID + | LIBUTF8_RETURN_INVALID)); +} + + +static void +check_codepoint(uint_least32_t codepoint) +{ + static const unsigned extra_flags[] = { + 0u, + LIBUTF8_ALLOW_31_BITS, + LIBUTF8_ALLOW_2BYTE_NULL, + LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_2BYTE_NULL, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_2BYTE_NULL | LIBUTF8_ALLOW_LONG_ENCODING, + LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_2BYTE_NULL | LIBUTF8_ALLOW_LONG_ENCODING + }; + + static size_t corruption = 0u; + + size_t flags_i, decode_avail, encode_len; + char buf[128]; + int overlong; + unsigned flags; + size_t q, r; + + for (flags_i = 0u; flags_i < sizeof(extra_flags) / sizeof(*extra_flags); flags_i++) { + flags = extra_flags[flags_i]; + + overlong = 0; + for (encode_len = 1u; encode_len <= 6u; encode_len++) { + if (!long_encode(buf, codepoint, encode_len)) + continue; + + for (decode_avail = 1u; decode_avail < encode_len + 4u; decode_avail++) + check(buf, codepoint, decode_avail, encode_len, flags, overlong); + + overlong = 1; + } + } + + if (codepoint >> 7) { + encode_len = libutf8_encode_measure(codepoint); + encode_len = encode_len ? encode_len : 6u; + long_encode(buf, codepoint, encode_len); + q = corruption / 3u; + r = corruption % 3u; + q %= encode_len - 1u; + buf[q + 1u] = (char)((unsigned char)buf[q + 1u] ^ ((r ^ 3u) << 6)); + corruption += 1u; + assert(libutf8_decode_mem(buf, sizeof(buf), &(uint_least32_t){0u}, ~0u) == 0u); + } +} + + +int +main(void) +{ + static const char *rejected[] = { + "\x80", + "\x80\x80", + "\x80\x80\x80", + "\x80\x80\x80\x80", + "\x80\x80\x80\x80\x80", + "\x80\x80\x80\x80\x80\x80", + "\x80\x80\x80\x80\x80\x80\x80", + "\x80\x80\x80\x80\x80\x80\x80\x80", + "\xFE", + "\xFE\x80", + "\xFE\x80\x80", + "\xFE\x80\x80\x80", + "\xFE\x80\x80\x80\x80", + "\xFE\x80\x80\x80\x80\x80", + "\xFE\x80\x80\x80\x80\x80\x80", + "\xFE\x80\x80\x80\x80\x80\x80\x80", + "\xFF", + "\xFF\x80", + "\xFF\x80\x80", + "\xFF\x80\x80\x80", + "\xFF\x80\x80\x80\x80", + "\xFF\x80\x80\x80\x80\x80", + "\xFF\x80\x80\x80\x80\x80\x80", + "\xFF\x80\x80\x80\x80\x80\x80\x80", + NULL + }; + + uint_least32_t cp, res; + char buf[128]; + size_t i; + + SET_UP_TEST(); + + assert(libutf8_decode_mem(buf, 0u, &cp, LIBUTF8_RETURN_INVALID | LIBUTF8_ALLOW_31_BITS) == 0u); + + for (cp = 0u; cp <= (uint_least32_t)LIBUTF8_ULTIMATE_CODEPOINT;) { + check_codepoint(cp); + if (!(++cp & 0x7FFFu)) + ARM_ALARM(); + } + + check_codepoint((uint_least32_t)0x110000ul); + check_codepoint((uint_least32_t)(1ul << 22)); + check_codepoint((uint_least32_t)(1ul << 23)); + check_codepoint((uint_least32_t)(1ul << 24)); + check_codepoint((uint_least32_t)(1ul << 25)); + check_codepoint((uint_least32_t)(1ul << 26)); + check_codepoint((uint_least32_t)(1ul << 27)); + check_codepoint((uint_least32_t)(1ul << 28)); + check_codepoint((uint_least32_t)(1ul << 29)); + check_codepoint((uint_least32_t)(1ul << 30)); + + check_codepoint((uint_least32_t)0x11FFFFul); + check_codepoint((uint_least32_t)((1ul << 22) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 23) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 24) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 25) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 26) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 27) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 28) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 29) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 30) - 1ul)); + check_codepoint((uint_least32_t)((1ul << 31) - 1ul)); + + check_codepoint((uint_least32_t)(1ul << 31)); + check_codepoint((uint_least32_t)0xFFFFFFFFul); + + for (i = 0u; rejected[i]; i++) { + res = NONE; + assert(libutf8_decode_mem(rejected[i], strlen(rejected[i]), &res, ~0u) == 0u); + assert(res == NONE); + assert(libutf8_decode_mem(rejected[i], strlen(rejected[i]), NULL, ~0u) == 0u); + } + + return 0; +} + + +# else + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + char encoded[6]; + uint_least32_t cp; + size_t n; + + n = libutf8_decode_mem((const char *)data, size, &cp, + LIBUTF8_RETURN_INVALID | LIBUTF8_ALLOW_31_BITS | LIBUTF8_ALLOW_LONG_ENCODING); + + if (n && n <= size) { + if (libutf8_decode_mem((const char *)data, size, &cp, LIBUTF8_RETURN_INVALID | LIBUTF8_ALLOW_31_BITS) == n) { + assert(libutf8_encode(encoded, cp) == n); + assert(!memcmp(encoded, data, n)); + } + } + + return 0; +} + + +# endif +#endif diff --git a/libutf8_encode.3 b/libutf8_encode.3 new file mode 100644 index 0000000..94bcb95 --- /dev/null +++ b/libutf8_encode.3 @@ -0,0 +1,70 @@ +.TH LIBUTF8_ENCODE 3 LIBUTF8 +.SH NAME +libutf8_encode - Encode a codepoint as UTF-8 + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +size_t \fBlibutf8_encode\fP(char *\fIout_buffer\fP, uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_encode () +function encodes +.I codepoint +as UTF-8 in +.IR Iout_buffer . +The buffer must have room for the number of bytes returned, up to six bytes +for all accept values of +.I codepoint +and up to four bytes for all codepoints valid within the Unicode standard. +.PP +The output is not NUL-terminated. + +.SH RETURN VALUE +The +.BR libutf8_encode () +function returns the number of bytes written, or zero if +.I codepoint +is unrepresentable. + +.SH ERRORS +The +.BR libutf8_encode () +function cannot fail per se, and never sets +.IR errno ; +.B EDOM +is an appropriate value when the function returns zero. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_encode () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH NOTES +UTF-8 has historically supported 31-bit codepoints. The +.BR libutf8_encode () +function does so as well, and follows regulare pattern, extended, +as it was historically, to support 31-bit integers using up to +6 bytes. Thus the function supports codepoints beyond +.BR LIBUTF8_ULTIMATE_CODEPOINT . + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_encode_measure (3), +.BR libutf8_decode (3) diff --git a/libutf8_encode.c b/libutf8_encode.c new file mode 100644 index 0000000..ab8567e --- /dev/null +++ b/libutf8_encode.c @@ -0,0 +1,60 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +size_t +libutf8_encode(char *out_buffer, uint_least32_t codepoint) +{ + size_t n = libutf8_encode_measure(codepoint); + ENCODE(n); + return n; +} + + +#else + + +int +main(void) +{ + /* Already mostly tested in libutf8_encode_measure */ + + uint_least32_t cp; + char buf[128]; + char ones[128]; + unsigned char bit; + size_t i, n; + + SET_UP_TEST(); + + memset(ones, 255, sizeof(ones)); + + for (cp = 0u; cp < (uint_least32_t)0x200000ul; cp++) { + memset(buf, 255, sizeof(buf)); + n = libutf8_encode(buf, cp); + assert(!memcmp(&buf[n], ones, sizeof(buf) - n)); + + for (i = 1u; i < n; i++) + assert(((unsigned char)buf[i] & 0xC0u) == 0x80u); + + if (n == 1u) { + assert(((unsigned char)buf[0u] & 0x80u) == 0x00u); + continue; + } + + bit = (unsigned char)0x80u; + for (i = 0u; i < n; i++, bit >>= 1) + assert((unsigned char)buf[0u] & bit); + assert(!((unsigned char)buf[0u] & bit)); + } + + memset(buf, 255, sizeof(buf)); + assert(libutf8_encode(buf, (uint_least32_t)1 << 31) == 0u); + assert(!memcmp(buf, ones, sizeof(buf))); + + return 0; +} + + +#endif diff --git a/libutf8_encode_measure.3 b/libutf8_encode_measure.3 new file mode 100644 index 0000000..e88a6a9 --- /dev/null +++ b/libutf8_encode_measure.3 @@ -0,0 +1,58 @@ +.TH LIBUTF8_ENCODE_MEASURE 3 LIBUTF8 +.SH NAME +libutf8_encode_measure - Get the number of bytes required to encode a codepoint + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +size_t \fBlibutf8_encode_measure\fP(uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_encode_measure () +function returns the number of bytes required to encode +.I codepoint +as UTF-8. It returns zero if the codepoint cannot be +represented in 31 bits. + +.SH RETURN VALUE +The +.BR libutf8_encode_measure () +function returns the number of bytes required to encode +.IR codepoint , +or zero if it is unrepresentable. +This value cannot exceed 6, and does not exceed 4 for +codepoint less or equal to +.BR LIBUTF8_ULTIMATE_CODEPOINT (3). + +.SH ERRORS +The +.BR libutf8_encode_measure () +function cannot fail per se, and never sets +.IR errno ; +.B EDOM +is an appropriate value when the function returns zero. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_encode_measure () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_encode (3) diff --git a/libutf8_encode_measure.c b/libutf8_encode_measure.c new file mode 100644 index 0000000..6bb2359 --- /dev/null +++ b/libutf8_encode_measure.c @@ -0,0 +1,84 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline size_t libutf8_encode_measure(uint_least32_t codepoint); + + +#else + + +#define LIMIT(BITS) ((uint_least32_t)(1ul << (BITS))) + + +static void +check(uint_least32_t cp, size_t n) +{ + char buf[128]; + uint_least32_t rcp; + + memset(buf, 255, sizeof(buf)); + + assert(libutf8_encode_measure(cp) == n); + assert(libutf8_encode(buf, cp) == n); + + if (!n) + return; + + buf[n] = '\0'; + rcp = 0u; + rcp = ~rcp; + + assert(libutf8_decode(buf, &rcp, LIBUTF8_RETURN_INVALID | LIBUTF8_ALLOW_31_BITS) == n); + assert(rcp == cp); +} + + +static void +check_range(uint_least32_t start, uint_least32_t end, size_t n) +{ + uint_least32_t cp = 0u; + unsigned i; + + check(start++, n); + check(start++, n); + check(start++, n); + check(start++, n); + + check(--end, n); + check(--end, n); + check(--end, n); + check(--end, n); + + for (i = 0u; i < 1000u; i++) { + cp ^= (uint_least32_t)((uint_least32_t)rand() << 0); + cp ^= (uint_least32_t)((uint_least32_t)rand() << 8); + cp ^= (uint_least32_t)((uint_least32_t)rand() << 16); + cp ^= (uint_least32_t)((uint_least32_t)rand() << 24); + cp %= (uint_least32_t)(end - start); + cp += start; + check(cp, n); + } +} + +int +main(void) +{ + uint_least32_t i = 0u; + + SET_UP_TEST(); + + for (; i < LIMIT(7); i++) check(i, 1u); + for (; i < LIMIT(6 * 1 + 5); i++) check(i, 2u); + for (; i < LIMIT(6 * 2 + 4); i++) check(i, 3u); + for (; i < LIMIT(6 * 3 + 3); i++) check(i, 4u); + check_range(i, LIMIT(6 * 4 + 2), 5u); + check_range(LIMIT(6 * 4 + 2), LIMIT(31), 6u); + check_range(LIMIT(31), (uint_least32_t)~(uint_least32_t)0, 0u); + + return 0; +} + + +#endif diff --git a/libutf8_is_continuation.3 b/libutf8_is_continuation.3 new file mode 100644 index 0000000..a8b9951 --- /dev/null +++ b/libutf8_is_continuation.3 @@ -0,0 +1,47 @@ +.TH LIBUTF8_IS_CONTINUATION 3 LIBUTF8 +.SH NAME +libutf8_is_continuation - Test whether a byte is a UTF-8 codepoint continuation byte + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +int \fBlibutf8_is_continuation\fP(char \fIc\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_is_continuation () +function tests whether a byte is a UTF-8 codepoint continuation byte. + +.SH RETURN VALUE +The +.BR libutf8_is_continuation () +function returns 1 if the predicate is true, and 0 otherwise. + +.SH ERRORS +The +.BR libutf8_is_continuation () +function cannot fail. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_is_continuation () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_is_invalid (3) diff --git a/libutf8_is_continuation.c b/libutf8_is_continuation.c new file mode 100644 index 0000000..ef80d75 --- /dev/null +++ b/libutf8_is_continuation.c @@ -0,0 +1,27 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline int libutf8_is_continuation(char c); + + +#else + + +int +main(void) +{ + unsigned i = 0u; + + SET_UP_TEST(); + + for (; i < 0x80u; i++) assert(libutf8_is_continuation((char)i) == 0); + for (; i < 0xC0u; i++) assert(libutf8_is_continuation((char)i) == 1); + for (; i < 0xFFu; i++) assert(libutf8_is_continuation((char)i) == 0); + + return 0; +} + + +#endif diff --git a/libutf8_is_invalid.3 b/libutf8_is_invalid.3 new file mode 100644 index 0000000..9641c5f --- /dev/null +++ b/libutf8_is_invalid.3 @@ -0,0 +1,51 @@ +.TH LIBUTF8_IS_INVALID 3 LIBUTF8 +.SH NAME +libutf8_is_invalid - Test whether a codepoint is invalid Unicode + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +int \fBlibutf8_is_invalid\fP(uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_is_invalid () +function tests whether a codepoint is invalid Unicode. + +.SH RETURN VALUE +The +.BR libutf8_is_invalid () +function returns 1 if the predicate is true, and 0 otherwise. + +.SH ERRORS +The +.BR libutf8_is_invalid () +function cannot fail. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_is_invalid () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7) +.BR libutf8_is_valid (3), +.BR libutf8_is_noncharacter (3), +.BR libutf8_is_surrogate (3), +.BR LIBUTF8_ULTIMATE_CODEPOINT (3), +.BR libutf8_is_continuation (3) diff --git a/libutf8_is_invalid.c b/libutf8_is_invalid.c new file mode 100644 index 0000000..fe0938b --- /dev/null +++ b/libutf8_is_invalid.c @@ -0,0 +1,36 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline int libutf8_is_invalid(uint_least32_t codepoint); + + +#else + + +int +main(void) +{ + unsigned long int i = 0u; + uint_least32_t cp; + + SET_UP_TEST(); + + for (; i <= 0x10FFFFul; i++) { + cp = (uint_least32_t)i; + if (libutf8_is_noncharacter(cp)) + assert(libutf8_is_invalid(cp) == 1); + else if (libutf8_is_surrogate(cp)) + assert(libutf8_is_invalid(cp) == 1); + else + assert(libutf8_is_invalid(cp) == 0); + } + for (; i < 0x120000ul; i++) + assert(libutf8_is_invalid((uint_least32_t)i) == 1); + + return 0; +} + + +#endif diff --git a/libutf8_is_noncharacter.3 b/libutf8_is_noncharacter.3 new file mode 100644 index 0000000..b9a657d --- /dev/null +++ b/libutf8_is_noncharacter.3 @@ -0,0 +1,53 @@ +.TH LIBUTF8_IS_NONCHARACTER 3 LIBUTF8 +.SH NAME +libutf8_is_noncharacter - Test whether a codepoint is a Unicode non-character codepoint + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +int \fBlibutf8_is_noncharacter\fP(uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_is_noncharacter () +function tests whether a codepoint is a Unicode non-character +codepoint, that is, codepoint that is reserved as invalid and +has no special meaning. +.PP +The result is unspecified for codepoints above +.BR LIBUTF8_ULTIMATE_CODEPOINT (3) +which conform to the pattern of non-character codepoints. + +.SH RETURN VALUE +The +.BR libutf8_is_noncharacter () +function returns 1 if the predicate is true, and 0 otherwise. + +.SH ERRORS +The +.BR libutf8_is_noncharacter () +function cannot fail. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_is_noncharacter () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_is_invalid (3) diff --git a/libutf8_is_noncharacter.c b/libutf8_is_noncharacter.c new file mode 100644 index 0000000..ca24e6c --- /dev/null +++ b/libutf8_is_noncharacter.c @@ -0,0 +1,33 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline int libutf8_is_noncharacter(uint_least32_t codepoint); + + +#else + + +int +main(void) +{ + unsigned long int i = 0u, j; + + SET_UP_TEST(); + + for (; i < 0xFDD0ul; i++) assert(libutf8_is_noncharacter((uint_least32_t)i) == 0); + for (; i < 0xFDF0ul; i++) assert(libutf8_is_noncharacter((uint_least32_t)i) == 1); + for (; i < 0xFFFEul; i++) assert(libutf8_is_noncharacter((uint_least32_t)i) == 0); + for (; i < 0x10000ul; i++) assert(libutf8_is_noncharacter((uint_least32_t)i) == 1); + for (; i < 0x10FFFFul; i += 0x10000ul) { + j = i; + for (; j < i + 0xFFFEul; j++) assert(libutf8_is_noncharacter((uint_least32_t)j) == 0); + for (; j < i + 0x10000ul; j++) assert(libutf8_is_noncharacter((uint_least32_t)j) == 1); + } + + return 0; +} + + +#endif diff --git a/libutf8_is_surrogate.3 b/libutf8_is_surrogate.3 new file mode 100644 index 0000000..6d76996 --- /dev/null +++ b/libutf8_is_surrogate.3 @@ -0,0 +1,47 @@ +.TH LIBUTF8_IS_SURROGATE 3 LIBUTF8 +.SH NAME +libutf8_is_surrogate - Test whether a codepoint is a UTF-16 surrogate + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +int \fBlibutf8_is_surrogate\fP(uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_is_surrogate () +function tests whether a codepoint is a UTF-16 surrogate. + +.SH RETURN VALUE +The +.BR libutf8_is_surrogate () +function returns 1 if the predicate is true, and 0 otherwise. + +.SH ERRORS +The +.BR libutf8_is_surrogate () +function cannot fail. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_is_surrogate () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_is_invalid (3) diff --git a/libutf8_is_surrogate.c b/libutf8_is_surrogate.c new file mode 100644 index 0000000..fd5c0a9 --- /dev/null +++ b/libutf8_is_surrogate.c @@ -0,0 +1,27 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline int libutf8_is_surrogate(uint_least32_t codepoint); + + +#else + + +int +main(void) +{ + unsigned long int i = 0u; + + SET_UP_TEST(); + + for (; i < 0xD800ul; i++) assert(libutf8_is_surrogate((uint_least32_t)i) == 0); + for (; i < 0xE000ul; i++) assert(libutf8_is_surrogate((uint_least32_t)i) == 1); + for (; i < 0x120000ul; i++) assert(libutf8_is_surrogate((uint_least32_t)i) == 0); + + return 0; +} + + +#endif diff --git a/libutf8_is_valid.3 b/libutf8_is_valid.3 new file mode 100644 index 0000000..8655c7c --- /dev/null +++ b/libutf8_is_valid.3 @@ -0,0 +1,47 @@ +.TH LIBUTF8_IS_VALID 3 LIBUTF8 +.SH NAME +libutf8_is_valid - Test whether a codepoint is valid Unicode + +.SH SYNOPSIS +.nf +#include <libutf8.h> + +int \fBlibutf8_is_valid\fP(uint_least32_t \fIcodepoint\fP); +.fi +.PP +Link with +.IR -lutf8 . + +.SH DESCRIPTION +The +.BR libutf8_is_valid () +function tests whether a codepoint is valid Unicode. + +.SH RETURN VALUE +The +.BR libutf8_is_valid () +function returns 1 if the predicate is true, and 0 otherwise. + +.SH ERRORS +The +.BR libutf8_is_valid () +function cannot fail. + +.SH ATTRIBUTES +For an explanation of the terms used in this section, see +.BR attributes (7). +.PP +.TS +allbox; +lb lb lb lb +l l l l. +Interface Thread safety Async-signal safety Async-cancel safety +T{ +.BR libutf8_is_valid () +T} MT-Safe AS-Safe AC-Safe +.TE +.sp + +.SH SEE ALSO +.BR libutf8 (7), +.BR libutf8_is_invalid (3) diff --git a/libutf8_is_valid.c b/libutf8_is_valid.c new file mode 100644 index 0000000..8275c34 --- /dev/null +++ b/libutf8_is_valid.c @@ -0,0 +1,29 @@ +/* See LICENSE file for copyright and license details. */ +#include "common.h" +#ifndef TEST + + +extern inline int libutf8_is_valid(uint_least32_t codepoint); + + +#else + + +int +main(void) +{ + unsigned long int i; + uint_least32_t cp; + + SET_UP_TEST(); + + for (i = 0u; i < 0x120000ul; i++) { + cp = (uint_least32_t)i; + assert(libutf8_is_valid(cp) == !libutf8_is_invalid(cp)); + } + + return 0; +} + + +#endif diff --git a/mk/linux.mk b/mk/linux.mk new file mode 100644 index 0000000..ad58f69 --- /dev/null +++ b/mk/linux.mk @@ -0,0 +1,6 @@ +LIBEXT = so +LIBFLAGS = -shared -Wl,-soname,lib$(LIB_NAME).$(LIBEXT).$(LIB_MAJOR) +LIBMAJOREXT = $(LIBEXT).$(LIB_MAJOR) +LIBMINOREXT = $(LIBEXT).$(LIB_VERSION) + +FIX_INSTALL_NAME = : diff --git a/mk/macos.mk b/mk/macos.mk new file mode 100644 index 0000000..4170e5f --- /dev/null +++ b/mk/macos.mk @@ -0,0 +1,6 @@ +LIBEXT = dylib +LIBFLAGS = -dynamiclib -Wl,-compatibility_version,$(LIB_MAJOR) -Wl,-current_version,$(LIB_VERSION) +LIBMAJOREXT = $(LIB_MAJOR).$(LIBEXT) +LIBMINOREXT = $(LIB_VERSION).$(LIBEXT) + +FIX_INSTALL_NAME = install_name_tool -id "$(PREFIX)/lib/libutf8.$(LIBMAJOREXT)" diff --git a/mk/windows.mk b/mk/windows.mk new file mode 100644 index 0000000..ed5ec8d --- /dev/null +++ b/mk/windows.mk @@ -0,0 +1,6 @@ +LIBEXT = dll +LIBFLAGS = -shared +LIBMAJOREXT = $(LIB_MAJOR).$(LIBEXT) +LIBMINOREXT = $(LIB_VERSION).$(LIBEXT) + +FIX_INSTALL_NAME = : |