Fix bug that caused (custom) hash functions with fixed tag size to be unsupported

Signed-off-by: Mattias Andrée <m@maandree.se>

1 files changed, 226 insertions, 6 deletions

diff --git a/librecrypt_add_algorithm.c b/librecrypt_add_algorithm.c
index 877e410..d561983 100644
--- a/librecrypt_add_algorithm.c
+++ b/librecrypt_add_algorithm.c
@@ -15,7 +15,7 @@ librecrypt_add_algorithm(char *out_buffer, size_t size, const char *augend,
 	size_t prefix1, prefix2, min, ret, len, phraselen;
 	size_t hashsize1, hashsize2;
 	char *phrase, pad;
-	int strict_pad, r_int, nul_term;
+	int strict_pad, r_int, nul_term, saved_errno;
 	const unsigned char *lut;
 	ssize_t r;
 
@@ -167,10 +167,12 @@ librecrypt_add_algorithm(char *out_buffer, size_t size, const char *augend,
 	/* Chain the hash algorithms: write `augment` and hash */
 	r = librecrypt_crypt(out_buffer, nul_term ? size + 1u : 0u, phrase, phraselen, augment, ctx);
 	if (r <= 0) {
+		saved_errno = errno;
 		librecrypt_wipe(phrase, phraselen);
 		free(phrase);
 		if (!r)
 			abort(); /* $covered$ (impossible) */
+		errno = saved_errno;
 		return -1;
 	}
 	if (ret > (size_t)(SSIZE_MAX - r)) {
@@ -192,6 +194,194 @@ librecrypt_add_algorithm(char *out_buffer, size_t size, const char *augend,
 #else
 
 
+#define ALPHABET "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+#define ALT_ALPHABET "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/"
+
+NONSTRING static const char encoding_lut[256u] = MAKE_ENCODING_LUT(ALPHABET);
+NONSTRING static const char alt_encoding_lut[256u] = MAKE_ENCODING_LUT(ALT_ALPHABET);
+
+static const unsigned char decoding_lut[256u] = {
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, 62, XX, XX, XX, 63,
+	52, 53, 54, 55, 56, 57, 58, 59, 60, 61, XX, XX, XX, XX, XX, XX,
+	XX,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
+	15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, XX, XX, XX, XX, XX,
+	XX, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+	41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX
+};
+
+static const unsigned char alt_decoding_lut[256u] = {
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, 62, XX, XX, XX, 63,
+	 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, XX, XX, XX, XX, XX, XX,
+	XX, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
+	51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, XX, XX, XX, XX, XX,
+	XX, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
+	25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX,
+	XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX
+};
+
+static unsigned
+rot4_is_algorithm(const char *settings, size_t len)
+{
+	if (len >= sizeof("$rot4$") - 1u)
+		if (!strncmp(settings, "$rot4$", sizeof("$rot4$") - 1u))
+			return 1u;
+	return 0u;
+}
+
+static unsigned
+trunc_is_algorithm(const char *settings, size_t len)
+{
+	if (len >= sizeof("$trunc$") - 1u)
+		if (!strncmp(settings, "$trunc$", sizeof("$trunc$") - 1u))
+			return 1u;
+	return 0u;
+}
+
+static int
+trunc_supported(const char *phrase, size_t len, int text, const char *settings,
+                size_t prefix, size_t *len_out)
+{
+	size_t i, digit, n = 0u, q, r;
+
+	(void) phrase;
+	(void) len;
+	(void) text;
+
+	if (prefix < sizeof("$trunc$") - 1u || memcmp(settings, "$trunc$", sizeof("$trunc$") - 1u))
+		return 0;
+
+	if (prefix == sizeof("$trunc$") - 1u) {
+		*len_out = 4u;
+		return 1;
+	}
+
+	if (settings[sizeof("$trunc$") - 1u] == '*') {
+		for (i = sizeof("$trunc$*") - 1u; i < prefix; i++) {
+			if ('0' > settings[i] || settings[i] > '9')
+				return 0;
+			digit = (size_t)(settings[i] - '0');
+			if (n > (SIZE_MAX - digit) / 10u)
+				return 0;
+			n = n * 10u + digit;
+		}
+	} else {
+		for (i = sizeof("$trunc$") - 1u; i < prefix; i++) {
+			if (settings[i] == '@')
+				break;
+			if (alt_decoding_lut[(unsigned char)settings[i]] == XX)
+				return 0;
+			n += 1u;
+		}
+		q = n / 4u;
+		r = n % 4u;
+		if (r == 1u)
+			return 0;
+		n = q * 3u + r;
+		if (r) {
+			n -= 1u;
+			for (; r < 4u && i < prefix; r++, i++)
+				if (settings[i] != '@')
+					break;
+			if (r != 4u || i != prefix)
+				return 0;
+		}
+	}
+	if (!n)
+		return 0;
+	*len_out = n;
+	return 1;
+}
+
+static int
+rot4_hash(char *restrict out_buffer, size_t size, const char *phrase, size_t len,
+          const char *settings, size_t prefix, LIBRECRYPT_CONTEXT *ctx)
+{
+	size_t i;
+
+	(void) settings;
+	(void) prefix;
+	(void) ctx;
+
+	for (i = 0u; i < 8u && i < size; i++) {
+		out_buffer[i] = '\0';
+		if (i < len)
+			out_buffer[i] = (char)((((int)phrase[i] & 0x0F) << 4) | (((int)phrase[i] >> 4) & 0x0F));
+	}
+	return 0;
+}
+
+static int
+trunc_hash(char *restrict out_buffer, size_t size, const char *phrase, size_t len,
+           const char *settings, size_t prefix, LIBRECRYPT_CONTEXT *ctx)
+{
+	size_t hash_size, i;
+
+	(void) ctx;
+
+	if (!trunc_supported(NULL, 0u, 1, settings, prefix, &hash_size)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	size = MIN(hash_size, size);
+	for (i = 0u; i < size; i++)
+		out_buffer[i] = i < len ? phrase[i] : '\0';
+
+	return 0;
+}
+
+static const struct librecrypt_algorithm rot4_algo = {
+	.is_algorithm = &rot4_is_algorithm,
+	.hash = &rot4_hash,
+	.encoding_lut = encoding_lut,
+	.decoding_lut = decoding_lut,
+	.hash_size = 8u,
+	.flexible_hash_size = 0,
+	.strict_pad = 1,
+	.pad = '#'
+};
+
+static const struct librecrypt_algorithm trunc_algo = {
+	.is_algorithm = &trunc_is_algorithm,
+	.hash = &trunc_hash,
+	.encoding_lut = alt_encoding_lut,
+	.decoding_lut = alt_decoding_lut,
+	.hash_size = 4u,
+	.flexible_hash_size = 1,
+	.strict_pad = 1,
+	.pad = '@'
+};
+
+#define TEST_PHRASE64_ROT4 "EiM0RVZneIk#"
+#define TEST_HASH_ROT4 "ITJDVGV2h5g#"
+#define TEST_HASH_TRUNC "4ycQhg@@"
+#define TEST_HASH_TRUNC6 "4ycQhlpD"
+#define TEST_HASH_TRUNC_ROT4 "ITJDVAAAAAA#"
+#define TEST_HASH_TRUNC6_ROT4 "ITJDVGV2AAA#"
+#define TEST_HASH_ROT4_TRUNC "8j93l0@@"
+#define TEST_HASH_ROT4_TRUNC6 "8j93l6lS"
+#define TEST_HASH_TRUNC_TRUNC6 "4ycQhg00"
+
+
 int
 main(void)
 {
@@ -200,6 +390,8 @@ main(void)
 #define HASH1 "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTYVWXYZ/+"
 #define ASTRA "*48"
 
+	const struct librecrypt_algorithm custom[] = {rot4_algo, trunc_algo};
+	LIBRECRYPT_CONTEXT *ctx = NULL;
 	char buf[1024], phrase[sizeof(buf)], expected[sizeof(buf)], pad;
 	size_t i, min, phraselen;
 	int strict_pad;
@@ -213,7 +405,7 @@ main(void)
 #define CHECK(AUGEND, AUGMENT, RESULT)\
 	do {\
 		CANARY_FILL(buf);\
-		r = librecrypt_add_algorithm(buf, sizeof(buf), (AUGEND), (AUGMENT), NULL);\
+		r = librecrypt_add_algorithm(buf, sizeof(buf), (AUGEND), (AUGMENT), ctx);\
 		EXPECT(r > 0);\
 		EXPECT((size_t)r == strlen(RESULT));\
 		assert((size_t)r < sizeof(buf) + 1u);\
@@ -223,7 +415,7 @@ main(void)
 		\
 		for (i = (size_t)r + 2u;; i--) {\
 			CANARY_FILL(buf);\
-			EXPECT(librecrypt_add_algorithm(buf, i, (AUGEND), (AUGMENT), NULL) == r);\
+			EXPECT(librecrypt_add_algorithm(buf, i, (AUGEND), (AUGMENT), ctx) == r);\
 			if (!i) {\
 				CANARY_CHECK(buf, 0u);\
 				break;\
@@ -234,13 +426,13 @@ main(void)
 			CANARY_CHECK(buf, min + 1u);\
 		}\
 		\
-		EXPECT(librecrypt_add_algorithm(NULL, 0u, (AUGEND), (AUGMENT), NULL) == r);\
+		EXPECT(librecrypt_add_algorithm(NULL, 0u, (AUGEND), (AUGMENT), ctx) == r);\
 		\
 		assert(sizeof(buf) > strlen(AUGEND));\
 		\
 		CANARY_FILL(buf);\
 		stpcpy(buf, (AUGEND));\
-		EXPECT(librecrypt_add_algorithm(buf, sizeof(buf), buf, (AUGMENT), NULL) == r);\
+		EXPECT(librecrypt_add_algorithm(buf, sizeof(buf), buf, (AUGMENT), ctx) == r);\
 		EXPECT(!buf[r]);\
 		EXPECT(!memcmp(buf, (RESULT), (size_t)r));\
 		n = strlen(AUGEND) + 1u;\
@@ -251,7 +443,7 @@ main(void)
 			CANARY_FILL(buf);\
 			stpcpy(buf, (AUGEND));\
 			n = strlen(AUGEND) + 1u;\
-			EXPECT(librecrypt_add_algorithm(buf, i, buf, (AUGMENT), NULL) == r);\
+			EXPECT(librecrypt_add_algorithm(buf, i, buf, (AUGMENT), ctx) == r);\
 			if (!i) {\
 				CANARY_CHECK(buf, n);\
 				break;\
@@ -374,6 +566,34 @@ main(void)
 	EXPECT(errno == ENOSYS);
 	CANARY_CHECK(buf, 0u);
 
+	ctx = librecrypt_create_context();
+	assert(ctx != NULL);
+
+	librecrypt_set_custom_algorithms(ctx, custom, ELEMSOF(custom));
+	CHECK("$rot4$", "$rot4$", "$rot4$>$rot4$");
+	CHECK("$rot4$", "$trunc$", "$rot4$>$trunc$");
+	CHECK("$rot4$", "$trunc$*6", "$rot4$>$trunc$*6");
+	CHECK("$trunc$", "$trunc$", "$trunc$>$trunc$");
+	CHECK("$trunc$", "$trunc$*6", "$trunc$>$trunc$*6");
+	CHECK("$trunc$", "$rot4$", "$trunc$>$rot4$");
+	CHECK("$trunc$*4", "$trunc$", "$trunc$*4>$trunc$");
+	CHECK("$trunc$*4", "$trunc$*6", "$trunc$*4>$trunc$*6");
+	CHECK("$trunc$*4", "$rot4$", "$trunc$*4>$rot4$");
+	CHECK("$trunc$*6", "$trunc$", "$trunc$*6>$trunc$");
+	CHECK("$trunc$*6", "$trunc$*6", "$trunc$*6>$trunc$*6");
+	CHECK("$trunc$*6", "$rot4$", "$trunc$*6>$rot4$");
+	CHECK("$rot4$"TEST_HASH_ROT4, "$rot4$", "$rot4$>$rot4$"TEST_PHRASE64_ROT4);
+	CHECK("$rot4$"TEST_HASH_ROT4, "$trunc$", "$rot4$>$trunc$"TEST_HASH_ROT4_TRUNC);
+	CHECK("$rot4$"TEST_HASH_ROT4, "$trunc$*6", "$rot4$>$trunc$"TEST_HASH_ROT4_TRUNC6);
+	CHECK("$trunc$"TEST_HASH_TRUNC6, "$trunc$", "$trunc$*6>$trunc$"TEST_HASH_TRUNC);
+	CHECK("$trunc$"TEST_HASH_TRUNC6, "$trunc$*6", "$trunc$*6>$trunc$"TEST_HASH_TRUNC6);
+	CHECK("$trunc$"TEST_HASH_TRUNC6, "$rot4$", "$trunc$*6>$rot4$"TEST_HASH_TRUNC6_ROT4);
+	CHECK("$trunc$"TEST_HASH_TRUNC, "$trunc$", "$trunc$*4>$trunc$"TEST_HASH_TRUNC);
+	CHECK("$trunc$"TEST_HASH_TRUNC, "$trunc$*6", "$trunc$*4>$trunc$"TEST_HASH_TRUNC_TRUNC6);
+	CHECK("$trunc$"TEST_HASH_TRUNC, "$rot4$", "$trunc$*4>$rot4$"TEST_HASH_TRUNC_ROT4);
+
+	librecrypt_free_context(ctx);
+
 	STOP_RESOURCE_TEST();
 	return 0;
 }