Add character data

Signed-off-by: Mattias Andrée <m@maandree.se>

6 files changed, 357 insertions, 30 deletions

diff --git a/Blocks.parse.c b/Blocks.parse.c
index a5a626a..d167b66 100644
--- a/Blocks.parse.c
+++ b/Blocks.parse.c
@@ -1,6 +1,13 @@
 /* See LICENSE file for copyright and license details. */
 #include "parse-common.c"
 
+#if defined(__clang__)
+# pragma clang diagnostic ignored "-Wunsafe-buffer-usage" /* broken */
+# pragma clang diagnostic ignored "-Wimplicit-void-ptr-cast" /* C++ */
+# pragma clang diagnostic ignored "-Wpre-c11-compat" /* C++ */
+# pragma clang diagnostic ignored "-Wdisabled-macro-expansion" /* libc */
+#endif
+
 
 struct block {
 	char *name;
@@ -23,7 +30,7 @@ parse_line(char *text, size_t lineno)
 
 	if (!isxdigit(*text)) {
 	malformat:
-		fprintf(stderr, "%s: line %zu in is malformatted\n", argv0, lineno);
+		fprintf(stderr, "%s: line %zu is malformatted\n", argv0, lineno);
 		exit(1);
 	}
 
diff --git a/Makefile b/Makefile
index c12d50b..12a75ed 100644
--- a/Makefile
+++ b/Makefile
@@ -86,13 +86,13 @@ UnicodeData.$(UNICODE_VERSION).txt:
 	$(DOWNLOAD) 'https://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt' > $@
 
 Scripts.txt.c: Scripts.$(UNICODE_VERSION).txt Scripts.parse
-	./Scripts.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@
+	./Scripts.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@ || (rm -f -- "$@"; ! :)
 
 Blocks.txt.c: Blocks.$(UNICODE_VERSION).txt Blocks.parse
-	./Blocks.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@
+	./Blocks.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@ || (rm -f -- "$@"; ! :)
 
 UnicodeData.txt.c: UnicodeData.$(UNICODE_VERSION).txt UnicodeData.parse
-	./UnicodeData.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@
+	./UnicodeData.parse < $(@:.txt.c=).$(UNICODE_VERSION).txt > $@ || (rm -f -- "$@"; ! :)
 
 install: libcmap.a libcmap.$(LIBEXT)
 	mkdir -p -- "$(DESTDIR)$(PREFIX)/lib"
diff --git a/Scripts.parse.c b/Scripts.parse.c
index 6f90bd7..6198444 100644
--- a/Scripts.parse.c
+++ b/Scripts.parse.c
@@ -74,7 +74,7 @@ parse_line(char *text, size_t lineno)
 
 	if (!isxdigit(*text)) {
 	malformat:
-		fprintf(stderr, "%s: line %zu in is malformatted\n", argv0, lineno);
+		fprintf(stderr, "%s: line %zu is malformatted\n", argv0, lineno);
 		exit(1);
 	}
 
diff --git a/UnicodeData.parse.c b/UnicodeData.parse.c
index 173fb6a..cab9d5e 100644
--- a/UnicodeData.parse.c
+++ b/UnicodeData.parse.c
@@ -1,15 +1,115 @@
 /* See LICENSE file for copyright and license details. */
 #include "parse-common.c"
 
+/* As of Unicode 18.0.0, splitting decomposition_mapping into 32-bit and 16-bit
+ * only saves just shy of 11 KB, therefore it has been decided to keep the API
+ * simple by always using 32 */
+
+
+#define LONGEST_DECOMPOSITION 18u /* Unicode makes this guarantee */
+
 
 struct segment {
 	uint32_t first, last;
 };
 
+struct sequence {
+	char *string;
+	size_t len;
+};
+
 
 static struct segment *segments = NULL;
 static size_t nsegments = 0u;
 static int ret = 0;
+static char **lines = NULL;
+static size_t nlines = 0u;
+static struct sequence *sequences = NULL;
+static size_t nsequences = 0u;
+
+
+#if defined(__GNUC__)
+__attribute__((__format__(gnu_printf, 1, 2)))
+#endif
+static void
+add_linef(const char *fmt, ...)
+{
+	va_list args1, args2;
+	int len, len2;
+	char *text;
+	va_start(args1, fmt);
+	va_copy(args2, args1);
+	len = vsnprintf(NULL, 0u, fmt, args2);
+	if (len < 0) {
+		fprintf(stderr, "%s: snprintf: %s\n", argv0, strerror(errno));
+		exit(1);
+	}
+	va_end(args2);
+	text = malloc((size_t)len + 1u);
+	if (!text) {
+		fprintf(stderr, "%s: malloc %zu: %s\n", argv0, (size_t)len + 1u, strerror(errno));
+		exit(1);
+	}
+	len2 = vsnprintf(text, (size_t)len + 1u, fmt, args1);
+	if (len2 < 0) {
+		fprintf(stderr, "%s: snprintf: %s\n", argv0, strerror(errno));
+		exit(1);
+	} else if (len2 != len) {
+		fprintf(stderr, "abort on line %i\n", __LINE__);
+		abort();
+	}
+	va_end(args1);
+	lines = realloc(lines, ++nlines * sizeof(*lines));
+	if (!lines) {
+		fprintf(stderr, "%s: realloc %zu: %s\n", argv0, nlines * sizeof(*lines), strerror(errno));
+		exit(1);
+	}
+	lines[nlines - 1u] = text;
+}
+
+
+static void
+add_sequence(const char *sequence, size_t len)
+{
+	sequences = realloc(sequences, ++nsequences * sizeof(*sequences));
+	if (!sequences) {
+		fprintf(stderr, "%s: realloc %zu: %s\n", argv0, nsequences * sizeof(*sequences), strerror(errno));
+		exit(1);
+	}
+	sequences[nsequences - 1u].string = strdup(sequence);
+	if (!sequences[nsequences - 1u].string) {
+		fprintf(stderr, "%s: strdup: %s\n", argv0, strerror(errno));
+		exit(1);
+	}
+	sequences[nsequences - 1u].len = len;
+}
+
+
+static const char *
+get_decomposition_type(const char *text)
+{
+	static char buf[64];
+	size_t i;
+
+	if (*text++ != '<') return NULL;
+	for (i = 0u; *text && *text != '>'; i++, text++) {
+		if (i == sizeof(buf)) return NULL;
+		if ('A' <= *text && *text <= 'Z') {
+			buf[i++] = '_';
+			if (i == sizeof(buf)) return NULL;
+			buf[i] = *text;
+		} else if ('a' <= *text && *text <= 'z') {
+			buf[i] = (char)(*text + ('A' - 'a'));
+		} else {
+			return NULL;
+		}
+	}
+
+	if (*text != '>') return NULL;
+	if (i == sizeof(buf)) return NULL;
+	buf[i] = '\0';
+	return buf;
+}
 
 
 static char *
@@ -28,38 +128,108 @@ split(char **text)
 static int
 output_character_data(char *text)
 {
-	char *field;
+	char *field, *numeric_value = NULL, *simple_uppercase_mapping;
+	const char *formatted;
+	size_t n, i;
 
 	if (!(field = split(&text))) return 0; /* codepoint */
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(".name = \"%s\"", field);
+	add_linef(".name = \"%s\"", field);
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(", .general_category = GENERAL_CATEGORY_%s", field);
+	add_linef(", .general_category = GENERAL_CATEGORY_%s", field);
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(", .canonical_combining_classes = %su", field);
+	add_linef(", .canonical_combining_classes = %su", field);
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(", .bidirectional_category = BIDIRECTIONAL_CATEGORY_%s", field);
+	add_linef(", .bidirectional_category = BIDIRECTIONAL_CATEGORY_%s", field);
 
-	if (!(field = split(&text))) return 0; /* TODO Character Decomposition Mapping */
-	if (!(field = split(&text))) return 0; /* TODO Decimal digit value */
-	if (!(field = split(&text))) return 0; /* TODO Digit value */
-	if (!(field = split(&text))) return 0; /* TODO Numeric value */
+	if (!(field = split(&text))) return 0;
+	if (!*field) {
+		add_linef(", .decomposition_type = LIBCMAP_DECOMPOSITION_TYPE_CANONICAL");
+		add_linef(", .decomposition_mapping_size = 0u");
+		add_linef(", .decomposition_mapping = NULL");
+	} else if (*field == '<') {
+		formatted = get_decomposition_type(field);
+		if (!formatted) return 0;
+		add_linef(", .decomposition_type = LIBCMAP_DECOMPOSITION_TYPE_%s", formatted);
+		field = strchr(field, '>');
+		if (!field++) return 0;
+		if (*field++ != ' ') return 0;
+		goto decomposition_mapping;
+	} else {
+		add_linef(", .decomposition_type = LIBCMAP_DECOMPOSITION_TYPE_CANONICAL");
+	decomposition_mapping:
+		n = 0u;
+		for (i = 0u;;) {
+			if (('0' > field[i] || field[i] > '9') && ('A' > field[i] || field[i] > 'F')) return 0;
+			while (('0' <= field[i] && field[i] <= '9') || ('A' <= field[i] && field[i] <= 'F')) i++;
+			n += 1u;
+			if (!field[i]) break;
+			if (field[i] != ' ') return 0;
+			field[i++] = '_';
+		}
+		if (n > LONGEST_DECOMPOSITION) return 0;
+		add_linef(", .decomposition_mapping_size = %zuu", n);
+		add_linef(", .decomposition_mapping = &sequences[DECOMPOSITION_%s]", field);
+		add_sequence(field, n);
+	}
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(", .bidirectional_mirrored = BIDIRECTIONAL_MIRRORED_%s", field);
+	if (*field && !numeric_value) {
+		numeric_value = field;
+		add_linef(", .numeric_type = LIBCMAP_NUMERIC_TYPE_DECIMAL");
+	}
 
 	if (!(field = split(&text))) return 0;
-	ret |= printf(", .unicode_1_name = \"%s\"", field);
+	if (*field && !numeric_value) {
+		numeric_value = field;
+		add_linef(", .numeric_type = LIBCMAP_NUMERIC_TYPE_DIGIT");
+	}
+
+	if (!(field = split(&text))) return 0;
+	if (*field && !numeric_value) {
+		numeric_value = field;
+		add_linef(", .numeric_type = LIBCMAP_NUMERIC_TYPE_NUMERIC");
+	}
+
+	if (numeric_value) {
+		add_linef(", .numeric_value = \"%s\"", numeric_value);
+	} else {
+		add_linef(", .numeric_type = LIBCMAP_NUMERIC_TYPE_NONE");
+		add_linef(", .numeric_value = NULL");
+	}
+
+	if (!(field = split(&text))) return 0;
+	add_linef(", .bidirectional_mirrored = BIDIRECTIONAL_MIRRORED_%s", field);
+
+	if (!(field = split(&text))) return 0;
+	add_linef(", .unicode_1_name = \"%s\"", field);
 
 	if (!(field = split(&text))) return 0; /* ISO 10646 comment field -- always empty as of Unicode 5.2.0 */
 
-	if (!(field = split(&text))) return 0; /* TODO Uppercase Mapping */
-	if (!(field = split(&text))) return 0; /* TODO Lowercase Mapping */
-	if (!(field = split(&text))) return 0; /* TODO Titlecase Mapping */
+	if (!(field = split(&text))) return 0;
+	simple_uppercase_mapping = field;
+	if (*field)
+		add_linef(", .simple_uppercase_mapping = UINT32_C(0x%s)", field);
+	else
+		add_linef(", .simple_uppercase_mapping = 0u");
+
+	if (!(field = split(&text))) return 0;
+	if (*field)
+		add_linef(", .simple_lowercase_mapping = UINT32_C(0x%s)", field);
+	else
+		add_linef(", .simple_lowercase_mapping = 0u");
+
+	if (!(field = split(&text))) return 0;
+	if (*field)
+		add_linef(", .simple_titlecase_mapping = UINT32_C(0x%s)", field);
+	else if (*simple_uppercase_mapping)
+		add_linef(", .simple_titlecase_mapping = UINT32_C(0x%s)", simple_uppercase_mapping);
+	else
+		add_linef(", .simple_titlecase_mapping = 0u");
 
 	return 1;
 }
@@ -73,7 +243,7 @@ parse_line(char *text, size_t lineno)
 
 	if (!isxdigit(*text)) {
 	malformat:
-		fprintf(stderr, "%s: line %zu in is malformatted\n", argv0, lineno);
+		fprintf(stderr, "%s: line %zu is malformatted\n", argv0, lineno);
 		exit(1);
 	}
 
@@ -85,7 +255,7 @@ parse_line(char *text, size_t lineno)
 
 	if (!nsegments) {
 	new_segment:
-		ret |= printf("static const struct libcmap_codepoint_data data_from_%04lx[] = {\n", codepoint);
+		add_linef("static const struct libcmap_codepoint_data data_from_%04lx[] = {\n", codepoint);
 		segments = realloc(segments, ++nsegments * sizeof(*segments));
 		if (!segments) {
 			fprintf(stderr, "%s: realloc %zu: %s\n", argv0, nsegments * sizeof(*segments), strerror(errno));
@@ -94,33 +264,139 @@ parse_line(char *text, size_t lineno)
 		segments[nsegments - 1u].first = (uint32_t)codepoint;
 		goto new_codepoint;
 	} else if (segments[nsegments - 1u].last + 1u == (uint32_t)codepoint) {
-		ret |= printf(",\n");
+		add_linef(",\n");
 	new_codepoint:
 		segments[nsegments - 1u].last = (uint32_t)codepoint;
-		ret |= printf("\t{");
+		add_linef("\t{");
 		if (!output_character_data(text))
 			goto malformat;
-		ret |= printf("}");
+		add_linef("}");
 	} else {
 		if ((uint32_t)codepoint < segments[nsegments - 1u].last) {
 			fprintf(stderr, "%s: UnicodeData is not sorted\n", argv0);
 			exit(1);
 		}
-		ret |= printf("\n};\n");
+		add_linef("\n};\n");
 		goto new_segment;
 	}
 }
 
 
 static int
+sequence_cmp(const void *a_, const void *b_)
+{
+	const struct sequence *a = a_, *b = b_;
+	return a->len > b->len ? -1 : a->len < b->len; /* longest first */
+}
+
+
+static int
+is_same_sequence(const uint32_t *a, const uint32_t *b, size_t n)
+{
+	while (n--)
+		if (a[n] != b[n])
+			return 0;
+	return 1;
+}
+
+
+static size_t
+find_subsequece(const uint32_t *seq, size_t seqlen, struct sequence *subseq, uint32_t decomp[LONGEST_DECOMPOSITION])
+{
+	uint32_t cp;
+	size_t i, n = 0u;
+	size_t off = 0u;
+
+	if (subseq->len > LONGEST_DECOMPOSITION) {
+		fprintf(stderr, "abort on line %i\n", __LINE__);
+		abort();
+	}
+	if (!subseq->len) {
+		fprintf(stderr, "abort on line %i\n", __LINE__);
+		abort();
+	}
+
+	cp = 0u;
+	for (i = 0;; i++) {
+		if ('0' <= subseq->string[i] && subseq->string[i] <= '9') {
+			cp = cp * 16u + (uint32_t)(subseq->string[i] - '0');
+		} else if ('A' <= subseq->string[i] && subseq->string[i] <= 'F') {
+			cp = cp * 16u + (uint32_t)(subseq->string[i] - 'A') + 10u;
+		} else if (subseq->string[i] == '_' || !subseq->string[i]) {
+			if (!cp) {
+				fprintf(stderr, "%s: a character U+0000 in its decomposition mapping\n", argv0);
+				exit(1);
+			}
+			decomp[n++] = cp;
+			cp = 0u;
+			if (!subseq->string[i])
+				break;
+		} else {
+			fprintf(stderr, "abort on line %i\n", __LINE__);
+			abort();
+		}
+	}
+
+	if (n != subseq->len) {
+		fprintf(stderr, "abort on line %i\n", __LINE__);
+		abort();
+	}
+
+	for (; off < seqlen; off++) {
+		n = subseq->len < seqlen - off ? subseq->len : seqlen - off;
+		if (is_same_sequence(decomp, &seq[off], n))
+			break;
+	}
+
+	return off;
+}
+
+
+static int
 output(void)
 {
-	size_t i;
+	uint32_t decomp[LONGEST_DECOMPOSITION];
+	uint32_t *seq = NULL;
+	size_t i, seqlen = 0u, off, end, ext;
 
 	if (!nsegments) {
 		fprintf(stderr, "%s: character list is empty\n", argv0);
 		exit(1);
 	}
+	if (!nsequences) {
+		fprintf(stderr, "%s: no character has a decomposition mapping\n", argv0);
+		exit(1);
+	}
+
+	qsort(sequences, nsequences, sizeof(*sequences), &sequence_cmp);
+	for (i = 0u; i < nsequences; i++) {
+		off = find_subsequece(seq, seqlen, &sequences[i], decomp);
+		ret |= printf("#define DECOMPOSITION_%s %zuu\n", sequences[i].string, off);
+		end = off + sequences[i].len;
+		ext = end > seqlen ? end - seqlen : 0u;
+		off = sequences[i].len - ext;
+		if (ext) {
+			seq = realloc(seq, (seqlen + ext) * sizeof(*seq));
+			if (!seq) {
+				fprintf(stderr, "%s: realloc %zu: %s\n", argv0, (seqlen + ext) * sizeof(*seq), strerror(errno));
+				exit(1);
+			}
+			memcpy(&seq[seqlen], &decomp[off], ext * sizeof(*seq));
+			seqlen += ext;
+		}
+		free(sequences[i].string);
+	}
+	free(sequences);
+	ret |= printf("static const uint32_t sequences[] = {\n");
+	for (i = 0u; i < seqlen; i++)
+		ret |= printf("\tUINT32_C(0x%04lX)%s\n", (unsigned long int)seq[i], i + 1u == seqlen ? "" : ",");
+	ret |= printf("};\n");
+
+	for (i = 0u; i < nlines; i++) {
+		ret |= printf("%s", lines[i]);
+		free(lines[i]);
+	}
+	free(lines);
 
 	ret |= printf("\n};\n");
 
diff --git a/libcmap.h b/libcmap.h
index 9c32c77..50d2146 100644
--- a/libcmap.h
+++ b/libcmap.h
@@ -117,6 +117,33 @@ enum libcmap_bidirectional_category {
 
 #define LIBCMAP_BIDIRECTIONAL_SUPERCATEGORY_MASK 0x7F00u
 
+enum libcmap_numeric_type {
+	LIBCMAP_NUMERIC_TYPE_NONE,
+	LIBCMAP_NUMERIC_TYPE_DECIMAL,
+	LIBCMAP_NUMERIC_TYPE_DIGIT,
+	LIBCMAP_NUMERIC_TYPE_NUMERIC
+};
+
+enum libcmap_decomposition_type {
+	LIBCMAP_DECOMPOSITION_TYPE_CANONICAL,
+	LIBCMAP_DECOMPOSITION_TYPE_FONT,
+	LIBCMAP_DECOMPOSITION_TYPE_NO_BREAK,
+	LIBCMAP_DECOMPOSITION_TYPE_INITIAL,
+	LIBCMAP_DECOMPOSITION_TYPE_MEDIAL,
+	LIBCMAP_DECOMPOSITION_TYPE_FINAL,
+	LIBCMAP_DECOMPOSITION_TYPE_ISOLATED,
+	LIBCMAP_DECOMPOSITION_TYPE_CIRCLE,
+	LIBCMAP_DECOMPOSITION_TYPE_SUPER,
+	LIBCMAP_DECOMPOSITION_TYPE_SUB,
+	LIBCMAP_DECOMPOSITION_TYPE_VERTICAL,
+	LIBCMAP_DECOMPOSITION_TYPE_WIDE,
+	LIBCMAP_DECOMPOSITION_TYPE_NARROW,
+	LIBCMAP_DECOMPOSITION_TYPE_SMALL,
+	LIBCMAP_DECOMPOSITION_TYPE_SQUARE,
+	LIBCMAP_DECOMPOSITION_TYPE_FRACTION,
+	LIBCMAP_DECOMPOSITION_TYPE_COMPAT
+};
+
 
 /**
  * Contiguous range of Unicode codepoints
@@ -200,11 +227,20 @@ struct libcmap_codepoint_data {
 	const char *name;
 
 	/* TODO documentation is needed */
-	enum libcmap_general_category general_category;
-	enum libcmap_bidirectional_category bidirectional_category;
-	uint16_t canonical_combining_classes;
-	uint16_t bidirectional_mirrored;
 	const char *unicode_1_name;
+	const char *numeric_value;
+	const uint32_t *decomposition_mapping;
+	enum libcmap_general_category general_category : 16;
+	enum libcmap_bidirectional_category bidirectional_category : 16;
+	uint32_t simple_uppercase_mapping : 21;
+	enum libcmap_numeric_type numeric_type : 4;
+	enum libcmap_decomposition_type decomposition_type : 7;
+	uint32_t simple_lowercase_mapping : 21;
+	uint32_t canonical_combining_classes : 11;
+	uint32_t simple_titlecase_mapping : 21;
+	uint32_t decomposition_mapping_size : 6;
+	uint32_t bidirectional_mirrored : 1;
+	uint32_t : 4;
 };
 
 
diff --git a/parse-common.c b/parse-common.c
index aeac342..49f6d49 100644
--- a/parse-common.c
+++ b/parse-common.c
@@ -2,12 +2,20 @@
 #include <ctype.h>
 #include <errno.h>
 #include <inttypes.h>
+#include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
+#if defined(__clang__)
+# pragma clang diagnostic ignored "-Wunsafe-buffer-usage" /* broken */
+# pragma clang diagnostic ignored "-Wimplicit-void-ptr-cast" /* C++ */
+# pragma clang diagnostic ignored "-Wpre-c11-compat" /* C++ */
+# pragma clang diagnostic ignored "-Wdisabled-macro-expansion" /* libc */
+#endif
+
 
 #define ULTIMATE_CODEPOINT 0x10FFFFul