diff options
| -rw-r--r-- | LICENSE | 2 | ||||
| -rw-r--r-- | Makefile | 20 | ||||
| -rw-r--r-- | behex_lower.c | 15 | ||||
| -rw-r--r-- | behex_upper.c | 15 | ||||
| -rw-r--r-- | config-armv8.mk | 23 | ||||
| -rw-r--r-- | config-portable.mk (renamed from config.mk) | 10 | ||||
| -rw-r--r-- | config-x86.mk | 20 | ||||
| -rw-r--r-- | hmac_marshal.c | 8 | ||||
| -rw-r--r-- | hmac_unmarshal.c | 10 | ||||
| -rw-r--r-- | libsha2.7 | 4 | ||||
| -rw-r--r-- | libsha2.h | 2 | ||||
| -rw-r--r-- | libsha2_algorithm_output_size.3 | 2 | ||||
| -rw-r--r-- | libsha2_behex_lower.3 | 2 | ||||
| -rw-r--r-- | libsha2_behex_upper.3 | 2 | ||||
| -rw-r--r-- | libsha2_digest.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_digest.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_init.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_marshal.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_state_output_size.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_unmarshal.3 | 2 | ||||
| -rw-r--r-- | libsha2_hmac_update.3 | 2 | ||||
| -rw-r--r-- | libsha2_init.3 | 2 | ||||
| -rw-r--r-- | libsha2_marshal.3 | 2 | ||||
| -rw-r--r-- | libsha2_state_output_size.3 | 2 | ||||
| -rw-r--r-- | libsha2_sum_fd.3 | 2 | ||||
| -rw-r--r-- | libsha2_unhex.3 | 2 | ||||
| -rw-r--r-- | libsha2_unmarshal.3 | 2 | ||||
| -rw-r--r-- | libsha2_update.3 | 2 | ||||
| -rw-r--r-- | marshal.c | 10 | ||||
| -rw-r--r-- | process.c | 266 | ||||
| -rw-r--r-- | sum_fd.c | 9 | ||||
| -rw-r--r-- | test.c | 4 | ||||
| -rw-r--r-- | unmarshal.c | 12 | ||||
| -rw-r--r-- | update.c | 4 |
34 files changed, 390 insertions, 78 deletions
@@ -1,6 +1,6 @@ ISC License -© 2015, 2019, 2022 Mattias Andrée <maandree@kth.se> +© 2015, 2019, 2022, 2026 Mattias Andrée <m@maandree.se> Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -1,6 +1,18 @@ .POSIX: -CONFIGFILE = config.mk +CONFIGFILE = config-x86.mk +# config-x86.mk is the default because it is compatible +# with and optimal for most machines, and if it is not +# supported, the compilation will immediately fail. It +# enables optimisations on x86 CPU's that have the +# required features. +# +# Additionally config-arm.mk which uses optimisations +# for ARMv8 is available. +# +# config-portable.mk is available for exotic CPU's +# and compiler that do not support the features required +# for the optimisations. include $(CONFIGFILE) OS = linux @@ -11,7 +23,7 @@ include mk/$(OS).mk LIB_MAJOR = 1 -LIB_MINOR = 0 +LIB_MINOR = 1 LIB_VERSION = $(LIB_MAJOR).$(LIB_MINOR) @@ -69,7 +81,7 @@ SRC = $(OBJ:.o=.c) all: libsha2.a libsha2.$(LIBEXT) test -$(OBJ): $(HDR) +$(OBJ) test.o: $(HDR) $(LOBJ): $(HDR) .c.o: @@ -90,7 +102,7 @@ libsha2.a: $(OBJ) $(AR) -s $@ check: test - ./test + $(CHECK_PREFIX) ./test $(CHECK_FLAGS) install: mkdir -p -- "$(DESTDIR)$(PREFIX)/lib" diff --git a/behex_lower.c b/behex_lower.c index 575ff39..93969ad 100644 --- a/behex_lower.c +++ b/behex_lower.c @@ -5,10 +5,21 @@ void libsha2_behex_lower(char *restrict output, const void *restrict hashsum_, size_t n) { +#define S(X)\ + X"0", X"1", X"2", X"3", X"4", X"5", X"6", X"7",\ + X"8", X"9", X"a", X"b", X"c", X"d", X"e", X"f" +#if defined(__GNUC__) + __attribute__((__nonstring__)) +#endif + static const char lut[256][2] = { + S("0"), S("1"), S("2"), S("3"), S("4"), S("5"), S("6"), S("7"), + S("8"), S("9"), S("a"), S("b"), S("c"), S("d"), S("e"), S("f") + }; + const unsigned char *restrict hashsum = hashsum_; output[2 * n] = '\0'; while (n--) { - output[2 * n + 0] = "0123456789abcdef"[(hashsum[n] >> 4) & 15]; - output[2 * n + 1] = "0123456789abcdef"[(hashsum[n] >> 0) & 15]; + output[2 * n + 0] = lut[hashsum[n]][0]; + output[2 * n + 1] = lut[hashsum[n]][1]; } } diff --git a/behex_upper.c b/behex_upper.c index cad6778..ac56218 100644 --- a/behex_upper.c +++ b/behex_upper.c @@ -5,10 +5,21 @@ void libsha2_behex_upper(char *restrict output, const void *restrict hashsum_, size_t n) { +#define S(X)\ + X"0", X"1", X"2", X"3", X"4", X"5", X"6", X"7",\ + X"8", X"9", X"A", X"B", X"C", X"D", X"E", X"F" +#if defined(__GNUC__) + __attribute__((__nonstring__)) +#endif + static const char lut[256][2] = { + S("0"), S("1"), S("2"), S("3"), S("4"), S("5"), S("6"), S("7"), + S("8"), S("9"), S("A"), S("B"), S("C"), S("D"), S("E"), S("F") + }; + const unsigned char *restrict hashsum = hashsum_; output[2 * n] = '\0'; while (n--) { - output[2 * n + 0] = "0123456789ABCDEF"[(hashsum[n] >> 4) & 15]; - output[2 * n + 1] = "0123456789ABCDEF"[(hashsum[n] >> 0) & 15]; + output[2 * n + 0] = lut[hashsum[n]][0]; + output[2 * n + 1] = lut[hashsum[n]][1]; } } diff --git a/config-armv8.mk b/config-armv8.mk new file mode 100644 index 0000000..47efdc2 --- /dev/null +++ b/config-armv8.mk @@ -0,0 +1,23 @@ +PREFIX = /usr +MANPREFIX = $(PREFIX)/share/man + +CC = cc -std=c11 + +COMMON_SANITIZE = -fsanitize=alignment,shift,signed-integer-overflow,object-size,null,undefined,bounds,address +CLANG_SANITIZE = -O1 $(COMMON_SANITIZE),cfi -flto -fvisibility=hidden -fno-sanitize-trap=cfi +GCC_SANITIZE = -O1 $(COMMON_SANITIZE) +#SANITIZE = $(CLANG_SANITIZE) +#SANITIZE = $(GCC_SANITIZE) + +CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 +CFLAGS = $(SANITIZE) -Wall -O3 -march=native +# If you cannot use -march=native, you should do e.g -march=armv8-a+crypto +# however, you have to be careful selecting the exact version, +# so you may have to replace armv8-a with something else. +LDFLAGS = $(SANITIZE) -s + +# You can add -DALLOCA_LIMIT=# to CPPFLAGS, where # is a size_t +# value, to put a limit on how large allocation the library is +# allowed to make with alloca(3). For buffers that can have any +# size this limit will be used if it wants to allocate a larger +# buffer. Choose 0 to use malloc(3) instead of alloca(3). diff --git a/config.mk b/config-portable.mk index a79e63f..a905df1 100644 --- a/config.mk +++ b/config-portable.mk @@ -3,9 +3,15 @@ MANPREFIX = $(PREFIX)/share/man CC = cc -std=c11 +COMMON_SANITIZE = -fsanitize=alignment,shift,signed-integer-overflow,object-size,null,undefined,bounds,address +CLANG_SANITIZE = -O1 $(COMMON_SANITIZE),cfi -flto -fvisibility=hidden -fno-sanitize-trap=cfi +GCC_SANITIZE = -O1 $(COMMON_SANITIZE) +#SANITIZE = $(CLANG_SANITIZE) +#SANITIZE = $(GCC_SANITIZE) + CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 -CFLAGS = -Wall -O3 -msse4 -msha -LDFLAGS = -s +CFLAGS = $(SANITIZE) -Wall -O3 +LDFLAGS = $(SANITIZE) -s # You can add -DALLOCA_LIMIT=# to CPPFLAGS, where # is a size_t # value, to put a limit on how large allocation the library is diff --git a/config-x86.mk b/config-x86.mk new file mode 100644 index 0000000..bd96527 --- /dev/null +++ b/config-x86.mk @@ -0,0 +1,20 @@ +PREFIX = /usr +MANPREFIX = $(PREFIX)/share/man + +CC = cc -std=c11 + +COMMON_SANITIZE = -fsanitize=alignment,shift,signed-integer-overflow,object-size,null,undefined,bounds,address +CLANG_SANITIZE = -O1 $(COMMON_SANITIZE),cfi -flto -fvisibility=hidden -fno-sanitize-trap=cfi +GCC_SANITIZE = -O1 $(COMMON_SANITIZE) +#SANITIZE = $(CLANG_SANITIZE) +#SANITIZE = $(GCC_SANITIZE) + +CPPFLAGS = -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_XOPEN_SOURCE=700 +CFLAGS = $(SANITIZE) -Wall -O3 -msse4 -msha +LDFLAGS = $(SANITIZE) -s + +# You can add -DALLOCA_LIMIT=# to CPPFLAGS, where # is a size_t +# value, to put a limit on how large allocation the library is +# allowed to make with alloca(3). For buffers that can have any +# size this limit will be used if it wants to allocate a larger +# buffer. Choose 0 to use malloc(3) instead of alloca(3). diff --git a/hmac_marshal.c b/hmac_marshal.c index 6db8972..e234f76 100644 --- a/hmac_marshal.c +++ b/hmac_marshal.c @@ -5,21 +5,21 @@ size_t libsha2_hmac_marshal(const struct libsha2_hmac_state *restrict state, void *restrict buf_) { - char *restrict buf = buf_; + unsigned char *restrict buf = buf_; size_t off = 0; if (buf) - *(int *)buf = 0; /* version */ + memcpy(buf, &(int){0}, sizeof(int)); /* version */ off += sizeof(int); off += libsha2_marshal(&state->sha2_state, buf ? &buf[off] : NULL); if (buf) - *(size_t *)&buf[off] = state->outsize; + memcpy(&buf[off], &state->outsize, sizeof(size_t)); off += sizeof(size_t); if (buf) - *(unsigned char *)&buf[off] = state->inited; + buf[off] = state->inited; off += sizeof(unsigned char); if (buf) diff --git a/hmac_unmarshal.c b/hmac_unmarshal.c index 197ea32..5ce2a5e 100644 --- a/hmac_unmarshal.c +++ b/hmac_unmarshal.c @@ -5,16 +5,18 @@ size_t libsha2_hmac_unmarshal(struct libsha2_hmac_state *restrict state, const void *restrict buf_, size_t bufsize) { - const char *restrict buf = buf_; + const unsigned char *restrict buf = buf_; size_t off = 0; size_t r; + int version; if (bufsize < sizeof(int)) { errno = EINVAL; return 0; } - if (*(const int *)buf) { /* version */ + memcpy(&version, buf, sizeof(int)); + if (version != 0) { errno = EINVAL; return 0; } @@ -30,10 +32,10 @@ libsha2_hmac_unmarshal(struct libsha2_hmac_state *restrict state, const void *re return 0; } - state->outsize = *(const size_t *)&buf[off]; + memcpy(&state->outsize, &buf[off], sizeof(size_t)); off += sizeof(size_t); - state->inited = *(const unsigned char *)&buf[off]; + state->inited = buf[off]; off += sizeof(unsigned char); memcpy(state->ipad, &buf[off], state->sha2_state.chunk_size); @@ -1,8 +1,8 @@ -.TH LIBSHA2 7 2022-07-07 libsha1 +.TH LIBSHA2 7 LIBSHA2 .SH NAME libsha2 \- SHA-2 hashing library .SH DESCRIPTION -.B libsha1 +.B libsha2 is an implementation of the SHA-2 family hashing functions: SHA-224, SHA-256, SHA-384, SHA-512, SHA-512/224, and SHA-512/256; with support for state marshalling and HMAC. @@ -67,7 +67,7 @@ struct libsha2_state { * For 32-bit algorithms */ uint_least32_t b32[64]; - + /** * For 64-bit algorithms */ diff --git a/libsha2_algorithm_output_size.3 b/libsha2_algorithm_output_size.3 index d083a38..32fb95a 100644 --- a/libsha2_algorithm_output_size.3 +++ b/libsha2_algorithm_output_size.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_ALGORITHM_OUTPUT_SIZE 3 2019-02-09 libsha2 +.TH LIBSHA2_ALGORITHM_OUTPUT_SIZE 3 LIBSHA2 .SH NAME libsha2_algorithm_output_size \- Get the size of the output for a SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_behex_lower.3 b/libsha2_behex_lower.3 index 1172397..e0ce17f 100644 --- a/libsha2_behex_lower.3 +++ b/libsha2_behex_lower.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_BEHEX_LOWER 3 2019-02-09 libsha2 +.TH LIBSHA2_BEHEX_LOWER 3 LIBSHA2 .SH NAME libsha2_behex_lower \- Convert binary to lower case hexadecimal .SH SYNOPSIS diff --git a/libsha2_behex_upper.3 b/libsha2_behex_upper.3 index c65202b..4d24b99 100644 --- a/libsha2_behex_upper.3 +++ b/libsha2_behex_upper.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_BEHEX_UPPER 3 2019-02-09 libsha2 +.TH LIBSHA2_BEHEX_UPPER 3 LIBSHA2 .SH NAME libsha2_behex_upper \- Convert binary to upper case hexadecimal .SH SYNOPSIS diff --git a/libsha2_digest.3 b/libsha2_digest.3 index 127171d..afd4d10 100644 --- a/libsha2_digest.3 +++ b/libsha2_digest.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_DIGEST 3 2019-02-09 libsha2 +.TH LIBSHA2_DIGEST 3 LIBSHA2 .SH NAME libsha2_digest \- Get the result of a SHA-2 hashing .SH SYNOPSIS diff --git a/libsha2_hmac_digest.3 b/libsha2_hmac_digest.3 index 1203049..faf6ab2 100644 --- a/libsha2_hmac_digest.3 +++ b/libsha2_hmac_digest.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_DIGEST 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_DIGEST 3 LIBSHA2 .SH NAME libsha2_hmac_digest \- Get the result of a HMAC-SHA-2 hashing .SH SYNOPSIS diff --git a/libsha2_hmac_init.3 b/libsha2_hmac_init.3 index 6f09680..bffe76c 100644 --- a/libsha2_hmac_init.3 +++ b/libsha2_hmac_init.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_INIT 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_INIT 3 LIBSHA2 .SH NAME libsha2_hmac_init \- Initialises hashing with an HMAC-SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_hmac_marshal.3 b/libsha2_hmac_marshal.3 index eca380c..48fe001 100644 --- a/libsha2_hmac_marshal.3 +++ b/libsha2_hmac_marshal.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_MARSHAL 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_MARSHAL 3 LIBSHA2 .SH NAME libsha2_hmac_marshal \- Marshal an HMAC-SHA-2 hashing state .SH SYNOPSIS diff --git a/libsha2_hmac_state_output_size.3 b/libsha2_hmac_state_output_size.3 index aa96436..93c2daf 100644 --- a/libsha2_hmac_state_output_size.3 +++ b/libsha2_hmac_state_output_size.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_STATE_OUTPUT_SIZE 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_STATE_OUTPUT_SIZE 3 LIBSHA2 .SH NAME libsha2_hmac_state_output_size \- Get the size of the output for a HMAC-SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_hmac_unmarshal.3 b/libsha2_hmac_unmarshal.3 index 6cc2967..467595c 100644 --- a/libsha2_hmac_unmarshal.3 +++ b/libsha2_hmac_unmarshal.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_UNMARSHAL 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_UNMARSHAL 3 LIBSHA2 .SH NAME libsha2_hmac_unmarshal \- Unmarshal an HMAC-SHA-2 hashing state .SH SYNOPSIS diff --git a/libsha2_hmac_update.3 b/libsha2_hmac_update.3 index bb02a18..7764c6e 100644 --- a/libsha2_hmac_update.3 +++ b/libsha2_hmac_update.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_HMAC_UPDATE 3 2019-02-10 libsha2 +.TH LIBSHA2_HMAC_UPDATE 3 LIBSHA2 .SH NAME libsha2_hmac_update \- Feed data into a HMAC-SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_init.3 b/libsha2_init.3 index 5823464..aca97b0 100644 --- a/libsha2_init.3 +++ b/libsha2_init.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_INIT 3 2019-02-09 libsha2 +.TH LIBSHA2_INIT 3 LIBSHA2 .SH NAME libsha2_init \- Initialises hashing with a SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_marshal.3 b/libsha2_marshal.3 index 4b44533..ecb5031 100644 --- a/libsha2_marshal.3 +++ b/libsha2_marshal.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_MARSHAL 3 2019-02-09 libsha2 +.TH LIBSHA2_MARSHAL 3 LIBSHA2 .SH NAME libsha2_marshal \- Marshal a SHA-2 hashing state .SH SYNOPSIS diff --git a/libsha2_state_output_size.3 b/libsha2_state_output_size.3 index 591eace..ef3130b 100644 --- a/libsha2_state_output_size.3 +++ b/libsha2_state_output_size.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_STATE_OUTPUT_SIZE 3 2019-02-09 libsha2 +.TH LIBSHA2_STATE_OUTPUT_SIZE 3 LIBSHA2 .SH NAME libsha2_state_output_size \- Get the size of the output for a SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_sum_fd.3 b/libsha2_sum_fd.3 index 8850e01..07e1864 100644 --- a/libsha2_sum_fd.3 +++ b/libsha2_sum_fd.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_SUM_FD 3 2019-02-09 libsha2 +.TH LIBSHA2_SUM_FD 3 LIBSHA2 .SH NAME libsha2_sum_fd \- Hash a file with a SHA-2 algorithm .SH SYNOPSIS diff --git a/libsha2_unhex.3 b/libsha2_unhex.3 index 7e009a0..5737057 100644 --- a/libsha2_unhex.3 +++ b/libsha2_unhex.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_UNHEX 3 2019-02-09 libsha2 +.TH LIBSHA2_UNHEX 3 LIBSHA2 .SH NAME libsha2_unhex \- Covert hexadecimal to binary .SH SYNOPSIS diff --git a/libsha2_unmarshal.3 b/libsha2_unmarshal.3 index 5df4929..a33eec5 100644 --- a/libsha2_unmarshal.3 +++ b/libsha2_unmarshal.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_UNMARSHAL 3 2019-02-09 libsha2 +.TH LIBSHA2_UNMARSHAL 3 LIBSHA2 .SH NAME libsha2_unmarshal \- Unmarshal a SHA-2 hashing state .SH SYNOPSIS diff --git a/libsha2_update.3 b/libsha2_update.3 index b7baf39..e64c972 100644 --- a/libsha2_update.3 +++ b/libsha2_update.3 @@ -1,4 +1,4 @@ -.TH LIBSHA2_UPDATE 3 2019-02-09 libsha2 +.TH LIBSHA2_UPDATE 3 LIBSHA2 .SH NAME libsha2_update \- Feed data into a SHA-2 algorithm .SH SYNOPSIS @@ -5,17 +5,17 @@ size_t libsha2_marshal(const struct libsha2_state *restrict state, void *restrict buf_) { - char *restrict buf = buf_; + unsigned char *restrict buf = buf_; size_t off = 0; if (buf) - *(int *)buf = 1; /* version */ + memcpy(buf, &(int){1}, sizeof(int)); off += sizeof(int); if (buf) - *(enum libsha2_algorithm *)&buf[off] = state->algorithm; + memcpy(&buf[off], &state->algorithm, sizeof(enum libsha2_algorithm)); off += sizeof(enum libsha2_algorithm); if (buf) - *(size_t *)&buf[off] = state->message_size; + memcpy(&buf[off], &state->message_size, sizeof(size_t)); off += sizeof(size_t); if (state->algorithm <= LIBSHA2_256) { @@ -35,7 +35,7 @@ libsha2_marshal(const struct libsha2_state *restrict state, void *restrict buf_) } if (buf) - *(size_t *)&buf[off] = state->chunk_size; + memcpy(&buf[off], &state->chunk_size, sizeof(size_t)); off += sizeof(size_t); if (buf) memcpy(&buf[off], state->chunk, (state->message_size / 8) % state->chunk_size); @@ -1,16 +1,35 @@ /* See LICENSE file for copyright and license details. */ #include "common.h" +#include <errno.h> #include <stdatomic.h> #if defined(__SSE4_1__) && defined(__SSSE3__) && defined(__SSE2__) && defined(__SHA__) -# define HAVE_X86_SHA_INTRINSICS +# define HAVE_X86_SHA256_INTRINSICS +#endif + +#if defined(__arm__) || defined(__aarch32__) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM) +# if defined(__ARM_NEON) && (defined(__ARM_ACLE) || defined(__ARM_FEATURE_CRYPTO)) +# define HAVE_ARM_SHA2_INTRINSICS +# endif #endif -#ifdef HAVE_X86_SHA_INTRINSICS +#ifdef HAVE_X86_SHA256_INTRINSICS # include <immintrin.h> #endif +#ifdef HAVE_ARM_SHA2_INTRINSICS +# include <asm/hwcap.h> +# include <sys/auxv.h> +# include <arm_neon.h> +# include <arm_acle.h> +enum sha2_intrinsics { + INTRINSICS_UNKNOWN = 0xFF, + SHA256_INTRINSICS = 0x01, + SHA512_INTRINSICS = 0x02 +}; +#endif + /** * Unified implementation (what can unified without performance impact) @@ -37,13 +56,13 @@ * @param h Hash values * @param work_h Space for temporary hash values */ -#define SHA2_IMPLEMENTATION(chunk, A, B, C, D, E, F, G, H, I, J, K, L, WORD_T, WORD_SIZE, TRUNC, k, w, h, work_h) \ +#define SHA2_IMPLEMENTATION(chunk, A, B, C, D, E, F, G, H, I, J, K, L, WORD_T, WORD_SIZE, TRUNC, k, w, h, work_h)\ memcpy(work_h, h, sizeof(work_h));\ \ memset(w, 0, 16 * sizeof(*(w)));\ for (i = 0; i < 16; i++)\ for (j = 0; j < WORD_SIZE; j++)\ - w[i] |= ((WORD_T)(chunk[(i + 1) * WORD_SIZE - j - 1])) << (j << 3);\ + w[i] |= (WORD_T)(((WORD_T)(chunk[(i + 1) * WORD_SIZE - j - 1])) << (j << 3));\ \ for (i = 16; i < sizeof(k) / sizeof(*(k)); i++) {\ w[i] = w[i - 16] + w[i - 7];\ @@ -68,7 +87,7 @@ h[i] = TRUNC(h[i] + work_h[i]); -#ifdef HAVE_X86_SHA_INTRINSICS +#ifdef HAVE_X86_SHA256_INTRINSICS static size_t process_x86_sha256(struct libsha2_state *restrict state, const unsigned char *restrict data, size_t len) @@ -79,8 +98,8 @@ process_x86_sha256(struct libsha2_state *restrict state, const unsigned char *re const unsigned char *restrict chunk; size_t off = 0; - temp = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)&state->h.b32[0]), 0xB1); - s1 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)&state->h.b32[4]), 0x1B); + temp = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)(const unsigned char *)&state->h.b32[0]), 0xB1); + s1 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)(const unsigned char *)&state->h.b32[4]), 0x1B); s0 = _mm_alignr_epi8(temp, s1, 8); s1 = _mm_blend_epi16(s1, temp, 0xF0); @@ -90,6 +109,11 @@ process_x86_sha256(struct libsha2_state *restrict state, const unsigned char *re abef_orig = s0; cdgh_orig = s1; +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wsign-conversion" +#endif + msg = _mm_loadu_si128((const __m128i *)&chunk[0]); msg0 = _mm_shuffle_epi8(msg, SHUFFLE_MASK); msg = _mm_add_epi32(msg0, _mm_set_epi64x(0xE9B5DBA5B5C0FBCFULL, 0x71374491428A2F98ULL)); @@ -226,6 +250,10 @@ process_x86_sha256(struct libsha2_state *restrict state, const unsigned char *re msg = _mm_shuffle_epi32(msg, 0x0E); s0 = _mm_sha256rnds2_epu32(s0, s1, msg); +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif + s0 = _mm_add_epi32(s0, abef_orig); s1 = _mm_add_epi32(s1, cdgh_orig); } @@ -247,14 +275,14 @@ __attribute__((__constructor__)) static int have_sha_intrinsics(void) { - static volatile int ret = -1; - static volatile atomic_flag spinlock = ATOMIC_FLAG_INIT; + static volatile int ret = -1; + static volatile atomic_flag spinlock = ATOMIC_FLAG_INIT; int a, b, c, d; if (ret != -1) return ret; - while (atomic_flag_test_and_set(&spinlock)); + while (atomic_flag_test_and_set(&spinlock)); if (ret != -1) goto out; @@ -281,6 +309,200 @@ out: #endif +#ifdef HAVE_ARM_SHA2_INTRINSICS + +static size_t +process_arm_sha256(struct libsha2_state *restrict state, const unsigned char *restrict data, size_t len) +{ + static const uint32_t rc[] = { + UINT32_C(0x428A2F98), UINT32_C(0x71374491), UINT32_C(0xB5C0FBCF), UINT32_C(0xE9B5DBA5), + UINT32_C(0x3956C25B), UINT32_C(0x59F111F1), UINT32_C(0x923F82A4), UINT32_C(0xAB1C5ED5), + UINT32_C(0xD807AA98), UINT32_C(0x12835B01), UINT32_C(0x243185BE), UINT32_C(0x550C7DC3), + UINT32_C(0x72BE5D74), UINT32_C(0x80DEB1FE), UINT32_C(0x9BDC06A7), UINT32_C(0xC19BF174), + UINT32_C(0xE49B69C1), UINT32_C(0xEFBE4786), UINT32_C(0x0FC19DC6), UINT32_C(0x240CA1CC), + UINT32_C(0x2DE92C6F), UINT32_C(0x4A7484AA), UINT32_C(0x5CB0A9DC), UINT32_C(0x76F988DA), + UINT32_C(0x983E5152), UINT32_C(0xA831C66D), UINT32_C(0xB00327C8), UINT32_C(0xBF597FC7), + UINT32_C(0xC6E00BF3), UINT32_C(0xD5A79147), UINT32_C(0x06CA6351), UINT32_C(0x14292967), + UINT32_C(0x27B70A85), UINT32_C(0x2E1B2138), UINT32_C(0x4D2C6DFC), UINT32_C(0x53380D13), + UINT32_C(0x650A7354), UINT32_C(0x766A0ABB), UINT32_C(0x81C2C92E), UINT32_C(0x92722C85), + UINT32_C(0xA2BFE8A1), UINT32_C(0xA81A664B), UINT32_C(0xC24B8B70), UINT32_C(0xC76C51A3), + UINT32_C(0xD192E819), UINT32_C(0xD6990624), UINT32_C(0xF40E3585), UINT32_C(0x106AA070), + UINT32_C(0x19A4C116), UINT32_C(0x1E376C08), UINT32_C(0x2748774C), UINT32_C(0x34B0BCB5), + UINT32_C(0x391C0CB3), UINT32_C(0x4ED8AA4A), UINT32_C(0x5B9CCA4F), UINT32_C(0x682E6FF3), + UINT32_C(0x748F82EE), UINT32_C(0x78A5636F), UINT32_C(0x84C87814), UINT32_C(0x8CC70208), + UINT32_C(0x90BEFFFA), UINT32_C(0xA4506CEB), UINT32_C(0xBEF9A3F7), UINT32_C(0xC67178F2) + }; + + uint32x4_t abcd, efgh, abcd_orig, efgh_orig; + uint32x4_t msg0, msg1, msg2, msg3, tmp0, tmp1, tmp2; + const unsigned char *restrict chunk; + size_t off = 0; + + abcd_orig = vld1q_u32(&state->h.b32[0]); + efgh_orig = vld1q_u32(&state->h.b32[4]); + + for (; len - off >= state->chunk_size; off += state->chunk_size) { + abcd = abcd_orig; + efgh = efgh_orig; + + chunk = &data[off]; + msg0 = vld1q_u32((const uint32_t *)&chunk[0]); + msg1 = vld1q_u32((const uint32_t *)&chunk[16]); + msg2 = vld1q_u32((const uint32_t *)&chunk[32]); + msg3 = vld1q_u32((const uint32_t *)&chunk[48]); + msg0 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(msg0))); + msg1 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(msg1))); + msg2 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(msg2))); + msg3 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(msg3))); + + tmp0 = vaddq_u32(msg0, vld1q_u32(&rc[0 * 4])); + + msg0 = vsha256su0q_u32(msg0, msg1); + tmp2 = abcd; + tmp1 = vaddq_u32(msg1, vld1q_u32(&rc[1 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg0 = vsha256su1q_u32(msg0, msg2, msg3); + + msg1 = vsha256su0q_u32(msg1, msg2); + tmp2 = abcd; + tmp0 = vaddq_u32(msg2, vld1q_u32(&rc[2 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg1 = vsha256su1q_u32(msg1, msg3, msg0); + + msg2 = vsha256su0q_u32(msg2, msg3); + tmp2 = abcd; + tmp1 = vaddq_u32(msg3, vld1q_u32(&rc[3 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg2 = vsha256su1q_u32(msg2, msg0, msg1); + + msg3 = vsha256su0q_u32(msg3, msg0); + tmp2 = abcd; + tmp0 = vaddq_u32(msg0, vld1q_u32(&rc[4 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg3 = vsha256su1q_u32(msg3, msg1, msg2); + + msg0 = vsha256su0q_u32(msg0, msg1); + tmp2 = abcd; + tmp1 = vaddq_u32(msg1, vld1q_u32(&rc[5 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg0 = vsha256su1q_u32(msg0, msg2, msg3); + + msg1 = vsha256su0q_u32(msg1, msg2); + tmp2 = abcd; + tmp0 = vaddq_u32(msg2, vld1q_u32(&rc[6 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg1 = vsha256su1q_u32(msg1, msg3, msg0); + + msg2 = vsha256su0q_u32(msg2, msg3); + tmp2 = abcd; + tmp1 = vaddq_u32(msg3, vld1q_u32(&rc[7 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg2 = vsha256su1q_u32(msg2, msg0, msg1); + + msg3 = vsha256su0q_u32(msg3, msg0); + tmp2 = abcd; + tmp0 = vaddq_u32(msg0, vld1q_u32(&rc[8 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg3 = vsha256su1q_u32(msg3, msg1, msg2); + + msg0 = vsha256su0q_u32(msg0, msg1); + tmp2 = abcd; + tmp1 = vaddq_u32(msg1, vld1q_u32(&rc[9 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg0 = vsha256su1q_u32(msg0, msg2, msg3); + + msg1 = vsha256su0q_u32(msg1, msg2); + tmp2 = abcd; + tmp0 = vaddq_u32(msg2, vld1q_u32(&rc[10 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg1 = vsha256su1q_u32(msg1, msg3, msg0); + + msg2 = vsha256su0q_u32(msg2, msg3); + tmp2 = abcd; + tmp1 = vaddq_u32(msg3, vld1q_u32(&rc[11 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + msg2 = vsha256su1q_u32(msg2, msg0, msg1); + + msg3 = vsha256su0q_u32(msg3, msg0); + tmp2 = abcd; + tmp0 = vaddq_u32(msg0, vld1q_u32(&rc[12 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + msg3 = vsha256su1q_u32(msg3, msg1, msg2); + + tmp2 = abcd; + tmp1 = vaddq_u32(msg1, vld1q_u32(&rc[13 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + + tmp2 = abcd; + tmp0 = vaddq_u32(msg2, vld1q_u32(&rc[14 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + + tmp2 = abcd; + tmp1 = vaddq_u32(msg3, vld1q_u32(&rc[15 * 4])); + abcd = vsha256hq_u32(abcd, efgh, tmp0); + efgh = vsha256h2q_u32(efgh, tmp2, tmp0); + + tmp2 = abcd; + abcd = vsha256hq_u32(abcd, efgh, tmp1); + efgh = vsha256h2q_u32(efgh, tmp2, tmp1); + + abcd_orig = vaddq_u32(abcd_orig, abcd); + efgh_orig = vaddq_u32(efgh_orig, efgh); + } + + vst1q_u32(&state->h.b32[0], abcd_orig); + vst1q_u32(&state->h.b32[4], efgh_orig); + + return off; +} + +# if defined(__GNUC__) +__attribute__((__constructor__)) +# endif +static enum sha2_intrinsics +have_sha_intrinsics(void) +{ + static volatile enum sha2_intrinsics ret = INTRINSICS_UNKNOWN; + static volatile atomic_flag spinlock = ATOMIC_FLAG_INIT; + unsigned long int caps; + enum sha2_intrinsics x; + int saved_errno; + + if (ret != INTRINSICS_UNKNOWN) + return ret; + + while (atomic_flag_test_and_set(&spinlock)); + + saved_errno = errno; + caps = getauxval(AT_HWCAP); + errno = saved_errno; + x = (caps & HWCAP_SHA2) ? SHA256_INTRINSICS : 0; + x |= (caps & HWCAP_SHA512) ? SHA512_INTRINSICS : 0; + ret = x; + + if (ret != INTRINSICS_UNKNOWN) + goto out; + +out: + atomic_flag_clear(&spinlock); + return ret; +} + +#endif size_t libsha2_process(struct libsha2_state *restrict state, const unsigned char *restrict data, size_t len) @@ -289,6 +511,11 @@ libsha2_process(struct libsha2_state *restrict state, const unsigned char *restr size_t off = 0; if (state->algorithm <= LIBSHA2_256) { +#if defined(HAVE_ARM_SHA2_INTRINSICS) + if (have_sha_intrinsics() & SHA256_INTRINSICS) + return process_arm_sha256(state, data, len); +#else +# define ROTR(X, N) TRUNC32(((X) >> (N)) | ((X) << (32 - (N)))) uint_least32_t s0, s1; size_t i, j; @@ -296,36 +523,33 @@ libsha2_process(struct libsha2_state *restrict state, const unsigned char *restr # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wmemset-elt-size" #endif -#define ROTR(X, N) TRUNC32(((X) >> (N)) | ((X) << (32 - (N)))) - -#ifdef HAVE_X86_SHA_INTRINSICS +# ifdef HAVE_X86_SHA256_INTRINSICS if (have_sha_intrinsics()) return process_x86_sha256(state, data, len); -#endif - +# endif for (; len - off >= state->chunk_size; off += state->chunk_size) { chunk = &data[off]; SHA2_IMPLEMENTATION(chunk, 7, 18, 3, 17, 19, 10, 6, 11, 25, 2, 13, 22, uint_least32_t, 4, TRUNC32, state->k.b32, state->w.b32, state->h.b32, state->work_h.b32); } - -#undef ROTR -#if defined(__GNUC__) -# pragma GCC diagnostic pop +# if defined(__GNUC__) +# pragma GCC diagnostic pop +# endif +# undef ROTR #endif } else { +#define ROTR(X, N) TRUNC64(((X) >> (N)) | ((X) << (64 - (N)))) uint_least64_t s0, s1; size_t i, j; -#define ROTR(X, N) TRUNC64(((X) >> (N)) | ((X) << (64 - (N)))) + /* TODO Add optimisation using ARMv8.2 SHA-512 intrinsics (when I've access to a machine supporting it) */ for (; len - off >= state->chunk_size; off += state->chunk_size) { chunk = &data[off]; SHA2_IMPLEMENTATION(chunk, 1, 8, 7, 19, 61, 6, 14, 18, 41, 28, 34, 39, uint_least64_t, 8, TRUNC64, state->k.b64, state->w.b64, state->h.b64, state->work_h.b64); } - #undef ROTR } @@ -18,15 +18,15 @@ libsha2_sum_fd(int fd, enum libsha2_algorithm algorithm, void *restrict hashsum) #ifndef _WIN32 if (fstat(fd, &attr) == 0 && attr.st_blksize > 0) - blksize = (size_t)(attr.st_blksize); + blksize = (size_t)attr.st_blksize; #endif #if ALLOCA_LIMIT > 0 if (blksize > (size_t)ALLOCA_LIMIT) { blksize = (size_t)ALLOCA_LIMIT; - blksize -= blksize % sizeof(((struct libsha2_state)NULL)->chunk); + blksize -= blksize % sizeof(((struct libsha2_state *)NULL)->chunk); if (!blksize) - blksize = sizeof(((struct libsha2_state)NULL)->chunk); + blksize = sizeof(((struct libsha2_state *)NULL)->chunk); } # if defined(__clang__) /* We are using a limit so it's just like declaring an array @@ -60,8 +60,9 @@ libsha2_sum_fd(int fd, enum libsha2_algorithm algorithm, void *restrict hashsum) } libsha2_digest(&state, NULL, 0, hashsum); + #if ALLOCA_LIMIT <= 0 - free(chunk); + free(chunk); #endif return 0; } @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) { - char buf[8096], str[2048]; + char buf[8192 * 32], str[2048]; struct libsha2_state s; struct libsha2_hmac_state hs; int skip_huge, fds[2], status; @@ -349,7 +349,9 @@ main(int argc, char *argv[]) exit(0); } close(fds[1]); + test(!errno); test(!libsha2_sum_fd(fds[0], LIBSHA2_256, buf)); + errno = 0; /* malloc(3) as been found to set errno=ENOENT on success */ test(waitpid(pid, &status, 0) == pid); test(!status); close(fds[0]); diff --git a/unmarshal.c b/unmarshal.c index 8c5780d..0bfecfd 100644 --- a/unmarshal.c +++ b/unmarshal.c @@ -5,7 +5,7 @@ size_t libsha2_unmarshal(struct libsha2_state *restrict state, const void *restrict buf_, size_t bufsize) { - const char *restrict buf = buf_; + const unsigned char *restrict buf = buf_; size_t off = 0; int version; @@ -14,16 +14,16 @@ libsha2_unmarshal(struct libsha2_state *restrict state, const void *restrict buf return 0; } - version = *(const int *)buf; - if (version < 0 || version > 1) { /* version */ + memcpy(&version, buf, sizeof(int)); + if (version < 0 || version > 1) { errno = EINVAL; return 0; } off += sizeof(int); - state->algorithm = *(const enum libsha2_algorithm *)&buf[off]; + memcpy(&state->algorithm, &buf[off], sizeof(enum libsha2_algorithm)); off += sizeof(enum libsha2_algorithm); - state->message_size = *(const size_t *)&buf[off]; + memcpy(&state->message_size, &buf[off], sizeof(size_t)); off += sizeof(size_t); switch (state->algorithm) { @@ -68,7 +68,7 @@ libsha2_unmarshal(struct libsha2_state *restrict state, const void *restrict buf errno = EINVAL; return 0; } - state->chunk_size = *(const size_t *)&buf[off]; + memcpy(&state->chunk_size, &buf[off], sizeof(size_t)); off += sizeof(size_t); if (bufsize - off < (state->message_size / 8) % state->chunk_size) { @@ -5,7 +5,7 @@ void libsha2_update(struct libsha2_state *restrict state, const void *restrict message_, size_t msglen) { - const char *restrict message = message_; + const unsigned char *restrict message = message_; size_t n, off; off = (state->message_size / 8) % state->chunk_size; @@ -21,7 +21,7 @@ libsha2_update(struct libsha2_state *restrict state, const void *restrict messag msglen -= n; } - off = libsha2_process(state, (const unsigned char *)message, msglen); + off = libsha2_process(state, message, msglen); if (msglen > off) memcpy(state->chunk, &message[off], msglen - off); |