Fix bug that caused (custom) hash functions with fixed tag size to be unsupported

Signed-off-by: Mattias Andrée <m@maandree.se>

1 files changed, 168 insertions, 5 deletions

diff --git a/librecrypt_test_supported.c b/librecrypt_test_supported.c
index f069d44..3e416c0 100644
--- a/librecrypt_test_supported.c
+++ b/librecrypt_test_supported.c
@@ -44,22 +44,171 @@ librecrypt_test_supported(const char *phrase, size_t len, int text, const char *
 # ifndef FUZZ
 
 
+#define ALPHABET "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+
+NONSTRING static const char encoding_lut[256u] = MAKE_ENCODING_LUT(ALPHABET);
+
+static const unsigned char decoding_lut[256u] = {
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, 62, XX, XX, XX, 63,
+	52, 53, 54, 55, 56, 57, 58, 59, 60, 61, XX, XX, XX, XX, XX, XX,
+	XX,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
+	15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, XX, XX, XX, XX, XX,
+	XX, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+	41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX
+};
+
+static unsigned
+rot4_is_algorithm(const char *settings, size_t len)
+{
+	if (len >= sizeof("$rot4$") - 1u)
+		if (!strncmp(settings, "$rot4$", sizeof("$rot4$") - 1u))
+			return 1u;
+	return 0u;
+}
+
+static unsigned
+trunc_is_algorithm(const char *settings, size_t len)
+{
+	if (len >= sizeof("$trunc$") - 1u)
+		if (!strncmp(settings, "$trunc$", sizeof("$trunc$") - 1u))
+			return 1u;
+	return 0u;
+}
+
+static int
+rot4_supported(const char *phrase, size_t len, int text, const char *settings,
+               size_t prefix, size_t *len_out)
+{
+	size_t i, j, n;
+
+	(void) phrase;
+	(void) len;
+	(void) text;
+
+	i = sizeof("$rot4$") - 1u;
+	if (prefix < i || memcmp(settings, "$rot4$", prefix))
+		return 0;
+	n = i - prefix;
+	if (n && n != 12u)
+		return 0;
+	if (n == 12u) {
+		for (j = 0u; j < 11u; j++, i++)
+			if (decoding_lut[(unsigned char)settings[i]] == XX)
+				return 0;
+		if (decoding_lut[(unsigned char)settings[i]] != '#')
+			return 0;
+	}
+
+	*len_out = 8u;
+	return 1;
+}
+
+static int
+trunc_supported(const char *phrase, size_t len, int text, const char *settings,
+                size_t prefix, size_t *len_out)
+{
+	size_t i, digit, n = 0u, q, r;
+
+	(void) phrase;
+	(void) len;
+	(void) text;
+
+	if (prefix < sizeof("$trunc$") - 1u || memcmp(settings, "$trunc$", sizeof("$trunc$") - 1u))
+		return 0;
+
+	if (prefix == sizeof("$trunc$") - 1u) {
+		*len_out = 4u;
+		return 1;
+	}
+
+	if (settings[sizeof("$trunc$") - 1u] == '*') {
+		for (i = sizeof("$trunc$*") - 1u; i < prefix; i++) {
+			if ('0' > settings[i] || settings[i] > '9')
+				return 0;
+			digit = (size_t)(settings[i] - '0');
+			if (n > (SIZE_MAX - digit) / 10u)
+				return 0;
+			n = n * 10u + digit;
+		}
+	} else {
+		for (i = sizeof("$trunc$") - 1u; i < prefix; i++) {
+			if (settings[i] == '@')
+				break;
+			if (decoding_lut[(unsigned char)settings[i]] == XX)
+				return 0;
+			n += 1u;
+		}
+		q = n / 4u;
+		r = n % 4u;
+		if (r == 1u)
+			return 0;
+		n = q * 3u + r;
+		if (r) {
+			n -= 1u;
+			for (; r < 4u && i < prefix; r++, i++)
+				if (settings[i] != '@')
+					break;
+			if (r != 4u || i != prefix)
+				return 0;
+		}
+	}
+	if (!n)
+		return 0;
+	*len_out = n;
+	return 1;
+}
+
+static const struct librecrypt_algorithm rot4_algo = {
+	.is_algorithm = &rot4_is_algorithm,
+	.test_supported = &rot4_supported,
+	.encoding_lut = encoding_lut,
+	.decoding_lut = decoding_lut,
+	.hash_size = 8u,
+	.flexible_hash_size = 0,
+	.strict_pad = 1,
+	.pad = '#'
+};
+
+static const struct librecrypt_algorithm trunc_algo = {
+	.is_algorithm = &trunc_is_algorithm,
+	.test_supported = &trunc_supported,
+	.encoding_lut = encoding_lut,
+	.decoding_lut = decoding_lut,
+	.hash_size = 4u,
+	.flexible_hash_size = 1,
+	.strict_pad = 1,
+	.pad = '@'
+};
+
 #define NSA "$~no~such~algorithm~$"
 
 
 #define CHECK(ALGO, VALID, INVALID)\
 	do {\
-		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID, NULL) == 1);\
-		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID">"NSA, NULL) == 0);\
-		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, NSA">"ALGO VALID, NULL) == 0);\
-		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID">"ALGO VALID, NULL) == 1);\
-		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO INVALID, NULL) == 0);\
+		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID, ctx) == 1);\
+		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID">"NSA, ctx) == 0);\
+		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, NSA">"ALGO VALID, ctx) == 0);\
+		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO VALID">"ALGO VALID, ctx) == 1);\
+		EXPECT(librecrypt_test_supported(NULL, 4096u, 0, ALGO INVALID, ctx) == 0);\
 	} while (0)
 
 
 int
 main(void)
 {
+	const struct librecrypt_algorithm custom[] = {rot4_algo, trunc_algo};
+	LIBRECRYPT_CONTEXT *ctx = NULL;
+
 	SET_UP_ALARM();
 	INIT_RESOURCE_TEST();
 
@@ -83,6 +232,20 @@ main(void)
 	IF__argon2id_v1_0__SUPPORTED(CHECK("$argon2id$", "m=8,t=1,p=1$*16$*40", "m=0,t=0,p=0$*1$*1"));
 	IF__argon2ds_v1_0__SUPPORTED(CHECK("$argon2ds$", "m=8,t=1,p=1$*16$*40", "m=0,t=0,p=0$*1$*1"));
 
+	ctx = librecrypt_create_context();
+	assert(ctx != NULL);
+
+	librecrypt_set_custom_algorithms(ctx, custom, ELEMSOF(custom));
+	CHECK("$rot4$", "", "x");
+	CHECK("$trunc$", "", "*0");
+	CHECK("$trunc$", "*6", "*0");
+	EXPECT(librecrypt_test_supported(NULL, 4096u, 0, "$rot4$>$trunc$", ctx) == 1);
+	EXPECT(librecrypt_test_supported(NULL, 4096u, 0, "$rot4$>$trunc$*8", ctx) == 1);
+	EXPECT(librecrypt_test_supported(NULL, 4096u, 0, "$trunc$*6>$rot4$", ctx) == 1);
+	EXPECT(librecrypt_test_supported(NULL, 4096u, 0, "$trunc$>$rot4$", ctx) == 1);
+
+	librecrypt_free_context(ctx);
+
 	STOP_RESOURCE_TEST();
 	return 0;
 }