diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index edac9b09603..d3afc016787 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -778,6 +778,7 @@ WC_HASH_CUSTOM_MAX_DIGEST_SIZE WC_HASH_CUSTOM_MIN_DIGEST_SIZE WC_INIT_ERROR_WHEN_CONTENDED WC_LINUXKM_NO_USE_HEAP_WRAPPERS +WC_LINUXKM_SVR_NO_BATCHING WC_MLDSA_NO_ASM WC_MLKEM_KERNEL_ASM WC_NOISE_SRC_MAX @@ -1275,6 +1276,7 @@ __ARM_NEON__ __ASSEMBLER__ __ATOMIC_CONSUME __ATOMIC_RELAXED +__ATOMIC_SEQ_CST __AVR_ARCH__ __AVR__ __AVX512F__ diff --git a/configure.ac b/configure.ac index edef8ca069d..6cf03f71f20 100644 --- a/configure.ac +++ b/configure.ac @@ -13190,7 +13190,7 @@ then fi -if test "$ENABLED_ARC4" == "yes" +if test "$ENABLED_ARC4" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ALLOW_RC4" fi diff --git a/linuxkm/linuxkm-fips-hash.c b/linuxkm/linuxkm-fips-hash.c index ff327f0ceb7..b3d3ad7d9ff 100644 --- a/linuxkm/linuxkm-fips-hash.c +++ b/linuxkm/linuxkm-fips-hash.c @@ -66,7 +66,7 @@ static int hmac_setkey_cb(Hmac *hmac, const byte *key, word32 key_len) { } static int hmac_update_cb(Hmac *hmac, const byte *in, word32 in_len) { - return wc_HmacUpdate_fips(hmac, in, in_len); + return wc_HmacUpdate(hmac, in, in_len); } static int hmac_final_cb(Hmac *hmac, byte *out, word32 out_sz) { diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index fdb42e2b5bd..051c37ed012 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -31,7 +31,7 @@ #include #include - #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error Unsupported kernel. #endif @@ -53,7 +53,8 @@ #endif #endif - #if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER_AESXTS) && defined(WC_LINUX_CONFIG_SELFTESTS_FULL) + #if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER_AESXTS) && defined(WC_LINUX_CONFIG_SELFTESTS_FULL) && \ + !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS expects AES-XTS-384 to work, even when CONFIG_CRYPTO_FIPS, but FIPS 140-3 only allows AES-XTS-256 and AES-XTS-512. */ #error CONFIG_CRYPTO_MANAGER_EXTRA_TESTS is incompatible with FIPS wolfCrypt AES-XTS -- please reconfigure the target kernel to disable CONFIG_CRYPTO_MANAGER_EXTRA_TESTS/CONFIG_CRYPTO_SELFTESTS_FULL. #endif @@ -70,7 +71,8 @@ defined(HAVE_ECC) && \ (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ defined(LINUXKM_LKCAPI_REGISTER_ECDSA) || \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_ECDSA))) + (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_ECDSA))) && \ + !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error Target kernel requires SHA-1 signature verification support. #endif @@ -125,7 +127,7 @@ * also needed to suppress inclusion of stdlib.h in * wolfssl/wolfcrypt/types.h. */ - #define XATOI(s) ({ \ + #define XATOI(s) __extension__ ({ \ long long _xatoi_res = 0; \ int _xatoi_ret = kstrtoll(s, 10, &_xatoi_res); \ if (_xatoi_ret != 0) { \ @@ -212,7 +214,7 @@ WOLFSSL_API int wc_linuxkm_sig_ignore_end(void); WOLFSSL_API int wc_linuxkm_check_for_intr_signals(void); #ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD - #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000 + #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD (25 * 1000 * 1000) #endif WOLFSSL_API void wc_linuxkm_relax_long_loop(void); @@ -833,12 +835,28 @@ #endif #endif #ifndef CAN_SAVE_VECTOR_REGISTERS - #define CAN_SAVE_VECTOR_REGISTERS() wc_can_save_vector_registers_x86() + #if defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) && \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) + #error Conflicting DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_* settings. + #elif defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) + #define CAN_SAVE_VECTOR_REGISTERS() 0 + #elif defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) + #define CAN_SAVE_VECTOR_REGISTERS() 1 + #else + #define CAN_SAVE_VECTOR_REGISTERS() wc_can_save_vector_registers_x86() + #endif #endif #if defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) && \ - defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) - #error Conflicting settings for DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_foo + defined(LINUXKM_LKCAPI_REGISTER) && \ + !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) + #error DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON is incompatible with LINUXKM_LKCAPI_REGISTER. + #endif + + #if defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) && \ + defined(WOLFSSL_LINUXKM_BENCHMARKS) && \ + !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) + #error DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF is incompatible with WOLFSSL_LINUXKM_BENCHMARKS. #endif #ifndef SAVE_VECTOR_REGISTERS @@ -879,7 +897,7 @@ #endif #ifndef SAVE_VECTOR_REGISTERS2 #if defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) - #define SAVE_VECTOR_REGISTERS2() \ + #define SAVE_VECTOR_REGISTERS2() __extension__ \ ({ \ int _svr_ret = wc_save_vector_registers_x86(WC_SVR_FLAG_NONE); \ if (_svr_ret != 0) { \ @@ -890,7 +908,10 @@ _svr_ret; \ }) #elif defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) - #define SAVE_VECTOR_REGISTERS2() WC_ACCEL_INHIBIT_E + #define SAVE_VECTOR_REGISTERS2() __extension__ ({ \ + WC_RELAX_LONG_LOOP(); \ + WC_ACCEL_INHIBIT_E; \ + }) #elif defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) #define SAVE_VECTOR_REGISTERS2() wc_save_vector_registers_x86(WC_SVR_FLAG_FUZZ) #else @@ -907,7 +928,7 @@ * DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON build -- ERROR if any * calls occur. */ - #define DISABLE_VECTOR_REGISTERS() \ + #define DISABLE_VECTOR_REGISTERS() __extension__ \ ({ \ pr_err("ERROR: DISABLE_VECTOR_REGISTERS() with DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON " \ "in %s at %s L %d\n", __func__, __FILE__, __LINE__); \ @@ -923,7 +944,10 @@ #endif #ifndef SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT - #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING + #if (defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) || \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF)) + #define SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT() SAVE_VECTOR_REGISTERS2() + #elif defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) #define SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT() wc_save_vector_registers_x86(WC_SVR_FLAG_FUZZ | WC_SVR_FLAG_MAYBE_INHIBIT) #else #define SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT() wc_save_vector_registers_x86(WC_SVR_FLAG_MAYBE_INHIBIT) @@ -969,7 +993,14 @@ #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm() #endif - #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) + #elif (defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ + (!defined(SAVE_VECTOR_REGISTERS) || \ + !defined(SAVE_VECTOR_REGISTERS2) || \ + !defined(RESTORE_VECTOR_REGISTERS) || \ + !defined(DISABLE_VECTOR_REGISTERS) || \ + !defined(REENABLE_VECTOR_REGISTERS) || \ + !defined(SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT) || \ + !defined(RESTORE_VECTOR_REGISTERS_MAYBE_INHIBITED))) #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ @@ -1325,7 +1356,7 @@ typeof(wc_linuxkm_free_svr_states) *wc_linuxkm_free_svr_states; typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86; typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86; - #else /* !CONFIG_X86 */ + #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 */ #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* arch */ @@ -1684,7 +1715,7 @@ #define wc_linuxkm_free_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_free_svr_states) #define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86) #define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86) - #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) + #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ @@ -1810,10 +1841,10 @@ #if defined(WOLFSSL_KERNEL_STACK_DEBUG) || defined(WC_LINUXKM_STACK_DEBUG) - #ifndef CONFIG_THREAD_INFO_IN_TASK + #if !defined(CONFIG_THREAD_INFO_IN_TASK) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error WC_LINUXKM_STACK_DEBUG requires CONFIG_THREAD_INFO_IN_TASK #endif - #ifdef CONFIG_STACK_GROWSUP + #if defined(CONFIG_STACK_GROWSUP) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error WC_LINUXKM_STACK_DEBUG requires !CONFIG_STACK_GROWSUP #endif @@ -2035,7 +2066,7 @@ #ifndef REENABLE_VECTOR_REGISTERS #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif - #else /* !CONFIG_X86 */ + #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 */ #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* !CONFIG_X86 */ #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ @@ -2059,7 +2090,7 @@ #ifdef WOLFSSL_LINUXKM_USE_MUTEXES #define WC_MUTEX_OPS_INLINE - #ifdef LINUXKM_LKCAPI_REGISTER + #if defined(LINUXKM_LKCAPI_REGISTER) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* must use spin locks when registering implementations with the * kernel, because mutexes are forbidden when calling with nonzero * irq_count(). @@ -2271,23 +2302,23 @@ #endif #ifdef WOLFSSL_TRACK_MEMORY - #define XMALLOC(s, h, t) ({(void)(h); (void)(t); wolfSSL_Malloc(s);}) + #define XMALLOC(s, h, t) __extension__ ({(void)(h); (void)(t); wolfSSL_Malloc(s);}) #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK - #define XFREE(p, h, t) ({(void)(h); (void)(t); wolfSSL_Free(p);}) + #define XFREE(p, h, t) __extension__ ({(void)(h); (void)(t); wolfSSL_Free(p);}) #else - #define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);}) + #define XFREE(p, h, t) __extension__ ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);}) #endif - #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);}) + #define XREALLOC(p, n, h, t) __extension__ ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);}) #else #if !defined(XMALLOC_USER) && !defined(XMALLOC_OVERRIDE) - #define XMALLOC(s, h, t) ({(void)(h); (void)(t); malloc(s);}) + #define XMALLOC(s, h, t) __extension__ ({(void)(h); (void)(t); malloc(s);}) #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK - #define XFREE(p, h, t) ({(void)(h); (void)(t); free(p);}) + #define XFREE(p, h, t) __extension__ ({(void)(h); (void)(t); free(p);}) #else - #define XFREE(p, h, t) ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);}) + #define XFREE(p, h, t) __extension__ ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);}) #endif #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC) - #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);}) + #define XREALLOC(p, n, h, t) __extension__ ({(void)(h); (void)(t); realloc(p, n);}) #endif #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */ #endif diff --git a/linuxkm/lkcapi_aes_glue.c b/linuxkm/lkcapi_aes_glue.c index 578c6210d3a..f72c14c9994 100644 --- a/linuxkm/lkcapi_aes_glue.c +++ b/linuxkm/lkcapi_aes_glue.c @@ -1170,6 +1170,24 @@ static int km_AesGcmSetAuthsize_Rfc4106(struct crypto_aead *tfm, unsigned int au typeof(wc_AesGcmEncryptUpdate_fips) wc_AesGcmEncryptUpdate; #endif +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_LINUXKM_SVR_NO_BATCHING) + #ifndef WC_LINUXKM_GCM_SVR_BATCH + #define WC_LINUXKM_GCM_SVR_BATCH (16 * 4096) + #endif + #if WC_LINUXKM_GCM_SVR_BATCH > 0 + /* If we're batching multiple chunks in a sequence wrapped in an outer + * SAVE_VECTOR_REGISTERS2(), we need to make sure the sk walk machinery + * doesn't try to yield. */ + #define WC_LINUXKM_GCM_WALK_ATOMIC true + #else + #define WC_LINUXKM_GCM_WALK_ATOMIC false + #endif +#else + #undef WC_LINUXKM_GCM_SVR_BATCH + #define WC_LINUXKM_GCM_SVR_BATCH 0 + #define WC_LINUXKM_GCM_WALK_ATOMIC false +#endif + static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) { struct crypto_aead * tfm = NULL; @@ -1182,6 +1200,9 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) u8 * assoc = NULL; u8 * assocmem = NULL; Aes *aes_copy = NULL; +#if WC_LINUXKM_GCM_SVR_BATCH > 0 + unsigned int svr_batch_left = 0; +#endif tfm = crypto_aead_reqtfm(req); ctx = crypto_aead_ctx(tfm); @@ -1199,10 +1220,10 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) scatterwalk_map_and_copy(authTag, req->src, req->assoclen + req->cryptlen - tfm->authsize, tfm->authsize, 0); - err = skcipher_walk_aead_decrypt(&walk, req, false); + err = skcipher_walk_aead_decrypt(&walk, req, WC_LINUXKM_GCM_WALK_ATOMIC); } else { - err = skcipher_walk_aead_encrypt(&walk, req, false); + err = skcipher_walk_aead_encrypt(&walk, req, WC_LINUXKM_GCM_WALK_ATOMIC); } if (unlikely(err)) { @@ -1218,6 +1239,16 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) goto out; } +#if WC_LINUXKM_GCM_SVR_BATCH > 0 + if (SAVE_VECTOR_REGISTERS2() == 0) { + svr_batch_left = WC_LINUXKM_GCM_SVR_BATCH; + /* all returns henceforth must be via the out: label. */ + } + /* else on failure, proceed unbatched -- per-call SVRs still work (or C + * fallback engages). + */ +#endif + #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 if (rfc4106_p) { byte rfc4106_iv[12]; @@ -1307,6 +1338,15 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) } while (walk.nbytes) { +#if WC_LINUXKM_GCM_SVR_BATCH > 0 + if (svr_batch_left == 0) { + if (SAVE_VECTOR_REGISTERS2() == 0) + svr_batch_left = WC_LINUXKM_GCM_SVR_BATCH; + /* else on failure, continue unbatched -- per-Update SVRs still work + * (or C fallback engages). + */ + } +#endif if (decrypt_p) { err = wc_AesGcmDecryptUpdate( aes_copy, @@ -1333,6 +1373,14 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) goto out; } +#if WC_LINUXKM_GCM_SVR_BATCH > 0 + if (svr_batch_left) { + svr_batch_left = (walk.nbytes >= svr_batch_left) ? 0 : svr_batch_left - walk.nbytes; + if (svr_batch_left == 0) + RESTORE_VECTOR_REGISTERS(); + } +#endif + err = skcipher_walk_done(&walk, 0); if (unlikely(err)) { @@ -1375,6 +1423,11 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) out: +#if WC_LINUXKM_GCM_SVR_BATCH > 0 + if (svr_batch_left) + RESTORE_VECTOR_REGISTERS(); +#endif + if (err && walk.nbytes) (void)skcipher_walk_done(&walk, err); @@ -2183,7 +2236,7 @@ static int ccmAesAead_rfc4309_loaded = 0; #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). #endif -#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) +#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) #error LKCAPI registration of AES-XTS with AESNI requires WC_C_DYNAMIC_FALLBACK. #endif @@ -2269,6 +2322,24 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key, typeof(wc_AesXtsEncryptUpdate_fips) wc_AesXtsEncryptUpdate; #endif +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_LINUXKM_SVR_NO_BATCHING) + #ifndef WC_LINUXKM_XTS_SVR_BATCH + #define WC_LINUXKM_XTS_SVR_BATCH (16 * 4096) + #endif + #if WC_LINUXKM_XTS_SVR_BATCH > 0 + /* If we're batching multiple chunks in a sequence wrapped in an outer + * SAVE_VECTOR_REGISTERS2(), we need to make sure the sk walk machinery + * doesn't try to yield. */ + #define WC_LINUXKM_XTS_WALK_ATOMIC true + #else + #define WC_LINUXKM_XTS_WALK_ATOMIC false + #endif +#else + #undef WC_LINUXKM_XTS_SVR_BATCH + #define WC_LINUXKM_XTS_SVR_BATCH 0 + #define WC_LINUXKM_XTS_WALK_ATOMIC false +#endif + static int km_AesXtsEncrypt(struct skcipher_request *req) { int err; @@ -2276,6 +2347,9 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) struct km_AesXtsCtx * ctx = NULL; struct skcipher_walk walk; unsigned int nbytes = 0; +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + unsigned int svr_batch_left = 0; +#endif tfm = crypto_skcipher_reqtfm(req); ctx = crypto_skcipher_ctx(tfm); @@ -2283,7 +2357,7 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) if (req->cryptlen < WC_AES_BLOCK_SIZE) return -EINVAL; - err = skcipher_walk_virt(&walk, req, false); + err = skcipher_walk_virt(&walk, req, WC_LINUXKM_XTS_WALK_ATOMIC); if (unlikely(err)) { pr_err("%s: skcipher_walk_virt failed: %d\n", @@ -2291,6 +2365,16 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) return err; } +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (SAVE_VECTOR_REGISTERS2() == 0) { + svr_batch_left = WC_LINUXKM_XTS_SVR_BATCH; + /* all returns henceforth must be via the out: label. */ + } + /* else on failure, proceed unbatched -- per-call SVRs still work (or C + * fallback engages). + */ +#endif + if (walk.nbytes == walk.total) { err = wc_AesXtsEncrypt(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes, walk.iv, walk.ivsize); @@ -2316,15 +2400,22 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) skcipher_request_set_tfm(&subreq, tfm); skcipher_request_set_callback(&subreq, +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + skcipher_request_flags(req) & ~CRYPTO_TFM_REQ_MAY_SLEEP, +#else skcipher_request_flags(req), +#endif NULL, NULL); skcipher_request_set_crypt(&subreq, req->src, req->dst, blocks * WC_AES_BLOCK_SIZE, req->iv); req = &subreq; - err = skcipher_walk_virt(&walk, req, false); - if (!walk.nbytes) - return err ? : -EINVAL; + err = skcipher_walk_virt(&walk, req, WC_LINUXKM_XTS_WALK_ATOMIC); + if (!walk.nbytes) { + if (! err) + err = -EINVAL; + goto out; + } } else { tail = 0; } @@ -2345,6 +2436,15 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) if (nbytes < walk.total) nbytes &= ~(WC_AES_BLOCK_SIZE - 1); +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left == 0) { + if (SAVE_VECTOR_REGISTERS2() == 0) + svr_batch_left = WC_LINUXKM_XTS_SVR_BATCH; + /* else on failure, proceed unbatched -- per-Update SVRs still + * work (or C fallback engages). + */ + } +#endif if (nbytes & ((unsigned int)WC_AES_BLOCK_SIZE - 1U)) err = wc_AesXtsEncryptFinal(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, nbytes, @@ -2366,8 +2466,16 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) if (unlikely(err)) { pr_err("%s: skcipher_walk_done failed: %d\n", crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); - return err; + goto out; } + +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left) { + svr_batch_left = (nbytes >= svr_batch_left) ? 0 : svr_batch_left - nbytes; + if (svr_batch_left == 0) + RESTORE_VECTOR_REGISTERS(); + } +#endif } if (unlikely(tail > 0)) { @@ -2381,9 +2489,9 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) skcipher_request_set_crypt(req, src, dst, WC_AES_BLOCK_SIZE + tail, req->iv); - err = skcipher_walk_virt(&walk, &subreq, false); + err = skcipher_walk_virt(&walk, &subreq, WC_LINUXKM_XTS_WALK_ATOMIC); if (err) - return err; + goto out; err = wc_AesXtsEncryptFinal(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes, @@ -2414,6 +2522,11 @@ static int km_AesXtsEncrypt(struct skcipher_request *req) out: +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left) + RESTORE_VECTOR_REGISTERS(); +#endif + if (err && walk.nbytes) (void)skcipher_walk_done(&walk, err); @@ -2427,6 +2540,9 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) struct km_AesXtsCtx * ctx = NULL; struct skcipher_walk walk; unsigned int nbytes = 0; +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + unsigned int svr_batch_left = 0; +#endif tfm = crypto_skcipher_reqtfm(req); ctx = crypto_skcipher_ctx(tfm); @@ -2434,7 +2550,7 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) if (req->cryptlen < WC_AES_BLOCK_SIZE) return -EINVAL; - err = skcipher_walk_virt(&walk, req, false); + err = skcipher_walk_virt(&walk, req, WC_LINUXKM_XTS_WALK_ATOMIC); if (unlikely(err)) { pr_err("%s: skcipher_walk_virt failed: %d\n", @@ -2442,6 +2558,16 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) return err; } +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (SAVE_VECTOR_REGISTERS2() == 0) { + svr_batch_left = WC_LINUXKM_XTS_SVR_BATCH; + /* all returns henceforth must be via the out: label. */ + } + /* else on failure, proceed unbatched -- per-call SVRs still work (or C + * fallback engages). + */ +#endif + if (walk.nbytes == walk.total) { err = wc_AesXtsDecrypt(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, @@ -2466,16 +2592,24 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) skcipher_walk_abort(&walk); skcipher_request_set_tfm(&subreq, tfm); - skcipher_request_set_callback(&subreq, - skcipher_request_flags(req), - NULL, NULL); + skcipher_request_set_callback( + &subreq, +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + skcipher_request_flags(req) & ~CRYPTO_TFM_REQ_MAY_SLEEP, +#else + skcipher_request_flags(req), +#endif + NULL, NULL); skcipher_request_set_crypt(&subreq, req->src, req->dst, blocks * WC_AES_BLOCK_SIZE, req->iv); req = &subreq; - err = skcipher_walk_virt(&walk, req, false); - if (!walk.nbytes) - return err ? : -EINVAL; + err = skcipher_walk_virt(&walk, req, WC_LINUXKM_XTS_WALK_ATOMIC); + if (!walk.nbytes) { + if (! err) + err = -EINVAL; + goto out; + } } else { tail = 0; } @@ -2496,6 +2630,14 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) if (nbytes < walk.total) nbytes &= ~(WC_AES_BLOCK_SIZE - 1); +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left == 0) { + if (SAVE_VECTOR_REGISTERS2() == 0) + svr_batch_left = WC_LINUXKM_XTS_SVR_BATCH; + /* else on failure, continue unbatched -- per-Update SVRs still + * work (or C fallback engages). */ + } +#endif if (nbytes & ((unsigned int)WC_AES_BLOCK_SIZE - 1U)) err = wc_AesXtsDecryptFinal(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, nbytes, @@ -2517,8 +2659,16 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) if (unlikely(err)) { pr_err("%s: skcipher_walk_done failed: %d\n", crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); - return err; + goto out; + } + +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left) { + svr_batch_left = (nbytes >= svr_batch_left) ? 0 : svr_batch_left - nbytes; + if (svr_batch_left == 0) + RESTORE_VECTOR_REGISTERS(); } +#endif } if (unlikely(tail > 0)) { @@ -2532,9 +2682,9 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) skcipher_request_set_crypt(req, src, dst, WC_AES_BLOCK_SIZE + tail, req->iv); - err = skcipher_walk_virt(&walk, &subreq, false); + err = skcipher_walk_virt(&walk, &subreq, WC_LINUXKM_XTS_WALK_ATOMIC); if (err) - return err; + goto out; err = wc_AesXtsDecryptFinal(ctx->aesXts, walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes, @@ -2565,6 +2715,11 @@ static int km_AesXtsDecrypt(struct skcipher_request *req) out: +#if WC_LINUXKM_XTS_SVR_BATCH > 0 + if (svr_batch_left) + RESTORE_VECTOR_REGISTERS(); +#endif + if (err && walk.nbytes) (void)skcipher_walk_done(&walk, err); diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index d4b9c70d0dd..685e4736010 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -37,12 +37,8 @@ * "fips_enabled" is only available in CONFIG_CRYPTO_FIPS kernels (otherwise * it's a macro hardcoding it to literal 0). */ - #if defined(CONFIG_CRYPTO_FIPS) != defined(HAVE_FIPS) - #ifdef HAVE_FIPS - #error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS (CONFIG_CRYPTO_FIPS unset). - #else - #error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS (HAVE_FIPS unset). - #endif + #if defined(HAVE_FIPS) && !defined(CONFIG_CRYPTO_FIPS) + #error wolfCrypt HAVE_FIPS with kernel CONFIG_CRYPTO_MANAGER requires kernel CONFIG_CRYPTO_FIPS. #endif #endif @@ -326,6 +322,13 @@ static int linuxkm_lkcapi_register(void) int seen_err = 0; int current_linuxkm_lkcapi_registering_now = 0; +#if defined(CONFIG_CRYPTO_FIPS) && !defined(HAVE_FIPS) + if (fips_enabled) { + pr_err("ERROR: can't load non-FIPS wolfCrypt module into fips_enabled kernel.\n"); + return -ECANCELED; + } +#endif + if (! wolfSSL_Atomic_Int_CompareExchange( &linuxkm_lkcapi_registering_now, ¤t_linuxkm_lkcapi_registering_now, diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index c661cbe65d4..44e6b72ea1f 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -63,32 +63,101 @@ enum linux_errcodes { my_EINVAL = EINVAL, my_ENOMEM = ENOMEM, - my_EBADMSG = EBADMSG + my_EBADMSG = EBADMSG, + my_ENOKEY = ENOKEY, + my_EFAULT = EFAULT, + my_EAFNOSUPPORT = EAFNOSUPPORT, + my_EOVERFLOW = EOVERFLOW, + my_EOPNOTSUPP = EOPNOTSUPP, + my_EDEADLK = EDEADLK, + my_EAGAIN = EAGAIN, + my_EBUSY = EBUSY, + my_ECANCELED = ECANCELED, + my_EINTR = EINTR, + my_ELIBBAD = ELIBBAD, + my_ENODATA = ENODATA, + my_ENODEV = ENODEV, + my_EPERM = EPERM, + my_EFBIG = EFBIG }; #undef EINVAL #undef ENOMEM #undef EBADMSG + #undef ENOKEY + #undef EFAULT + #undef EAFNOSUPPORT + #undef EOVERFLOW + #undef EOPNOTSUPP + #undef EDEADLK + #undef EAGAIN + #undef EBUSY + #undef ECANCELED + #undef EINTR + #undef ELIBBAD + #undef ENODATA + #undef ENODEV + #undef EPERM + #undef EFBIG #define EINVAL WC_ERR_TRACE(my_EINVAL) #define ENOMEM WC_ERR_TRACE(my_ENOMEM) #define EBADMSG WC_ERR_TRACE(my_EBADMSG) + #define ENOKEY WC_ERR_TRACE(my_ENOKEY) + #define EFAULT WC_ERR_TRACE(my_EFAULT) + #define EAFNOSUPPORT WC_ERR_TRACE(my_EAFNOSUPPORT) + #define EOVERFLOW WC_ERR_TRACE(my_EOVERFLOW) + #define EOPNOTSUPP WC_ERR_TRACE(my_EOPNOTSUPP) + #define EDEADLK WC_ERR_TRACE(my_EDEADLK) + #define EAGAIN WC_ERR_TRACE(my_EAGAIN) + #define EBUSY WC_ERR_TRACE(my_EBUSY) + #define ECANCELED WC_ERR_TRACE(my_ECANCELED) + #define EINTR WC_ERR_TRACE(my_EINTR) + #define ELIBBAD WC_ERR_TRACE(my_ELIBBAD) + #define ENODATA WC_ERR_TRACE(my_ENODATA) + #define ENODEV WC_ERR_TRACE(my_ENODEV) + #define EPERM WC_ERR_TRACE(my_EPERM) + #define EFBIG WC_ERR_TRACE(my_EFBIG) +#endif + +#ifdef CONFIG_HAVE_KPROBES + static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name); + #define WC_LINUXKM_HAVE_MY_KALLSYMS_LOOKUP_NAME +#endif + +static int wolfcrypt_inited; +#ifndef WOLFCRYPT_ONLY +static int wolfssl_inited; #endif static int libwolfssl_cleanup(void) { int ret; -#ifdef WOLFCRYPT_ONLY - ret = wolfCrypt_Cleanup(); + +#ifndef WOLFCRYPT_ONLY + if (wolfssl_inited) { + wolfssl_inited = 0; + ret = wolfSSL_Cleanup(); + if (ret != WOLFSSL_SUCCESS) + pr_err("ERROR: wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret)); + else + pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); + } +#endif + + if (wolfcrypt_inited) { + wolfcrypt_inited = 0; + ret = wolfCrypt_Cleanup(); + } + else + ret = 0; + if (ret != 0) pr_err("ERROR: wolfCrypt_Cleanup() failed: %s\n", wc_GetErrorString(ret)); else pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); -#else - ret = wolfSSL_Cleanup(); - if (ret != WOLFSSL_SUCCESS) - pr_err("ERROR: wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret)); - else - pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); + +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(HAVE_FIPS) + wc_linuxkm_free_svr_states(); #endif return ret; @@ -152,7 +221,8 @@ extern const unsigned int wolfCrypt_FIPS_ro_end[]; #endif /* WC_SYM_RELOC_TABLES */ -#ifdef HAVE_FIPS +#if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) + static void lkmFipsCb(int ok, int err, const char* hash) { if ((! ok) || (err != 0)) @@ -173,7 +243,7 @@ static void lkmFipsCb(int ok, int err, const char* hash) #ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE static int updateFipsHash(void); #endif -#endif /* HAVE_FIPS */ +#endif /* HAVE_FIPS && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef WOLFSSL_LINUXKM_BENCHMARKS extern int wolfcrypt_benchmark_main(int argc, char** argv); @@ -192,7 +262,14 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m) m->irq_flags = irq_flags; return 0; } + if (in_nmi()) + return BUSY_E; if (! wc_linuxkm_can_block()) { +#if IS_ENABLED(CONFIG_PREEMPT_RT) + /* RT spinlock_t is a sleeping rtmutex; an atomic caller has no legal + * wait -- the entry trylock was its one shot. */ + return BUSY_E; +#else /* !CONFIG_PREEMPT_RT */ /* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed as * required, because in_softirq() is always true while saved, * even for WC_FPU_INHIBITED_FLAG contexts. @@ -204,6 +281,7 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m) spin_lock_irqsave(&m->lock, irq_flags); m->irq_flags = irq_flags; return 0; +#endif /* !CONFIG_PREEMPT_RT */ } else { for (;;) { @@ -211,6 +289,11 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m) if (sig_ret) return sig_ret; cond_resched(); + /* FWIW, on PREEMPT_RT kernels, this polls a sleeping lock instead + * of blocking on it, which bypasses priority inheritance: legal, + * but a low-priority holder under a high-priority RT poller + * inherits nothing and the loop degrades to prioritized + * busy-wait. */ if (spin_trylock_irqsave(&m->lock, irq_flags)) { m->irq_flags = irq_flags; return 0; @@ -221,12 +304,25 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m) } #endif -WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_install_node(struct kobj_attribute *node, int *installed_flag) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0) + /* linux "module: Constify 'struct module_attribute'" (v6.14-rc1) */ + #define WC_MODULE_ATTR_CONST const +#else + #define WC_MODULE_ATTR_CONST +#endif + +/* Note, nodes on THIS_MODULE->mkobj.kobj are dispatched by module_sysfs_ops + * (see module_ktype in kernel/params.c), so their handlers must be typed as + * struct module_attribute callbacks, not struct kobj_attribute callbacks -- + * the latter only work by layout coincidence, and trap on kernels with + * CONFIG_CFI_CLANG. + */ +WC_MAYBE_UNUSED static int linuxkm_sysfs_install_attr(struct attribute *attr, int *installed_flag) { if ((installed_flag == NULL) || (! *installed_flag)) { - int ret = sysfs_create_file(&THIS_MODULE->mkobj.kobj, &node->attr); + int ret = sysfs_create_file(&THIS_MODULE->mkobj.kobj, attr); if (ret) { - pr_err("ERROR: sysfs_create_file failed for %s: %d\n", node->attr.name, ret); + pr_err("ERROR: sysfs_create_file failed for %s: %d\n", attr->name, ret); return ret; } if (installed_flag) @@ -235,16 +331,30 @@ WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_install_node(struct kobj_attribu return 0; } -WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attribute *node, int *installed_flag) +WC_MAYBE_UNUSED static int linuxkm_sysfs_deinstall_attr(struct attribute *attr, int *installed_flag) { if ((installed_flag == NULL) || *installed_flag) { - sysfs_remove_file(&THIS_MODULE->mkobj.kobj, &node->attr); + sysfs_remove_file(&THIS_MODULE->mkobj.kobj, attr); if (installed_flag) *installed_flag = 0; } return 0; } +/* Transitional wrappers for not-yet-converted struct kobj_attribute callers. + * Remove these, and the callers' CFI-incompatible handler typing, by + * converting the callers to struct module_attribute per the pattern above. + */ +WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_install_node(struct kobj_attribute *node, int *installed_flag) +{ + return linuxkm_sysfs_install_attr(&node->attr, installed_flag); +} + +WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attribute *node, int *installed_flag) +{ + return linuxkm_sysfs_deinstall_attr(&node->attr, installed_flag); +} + #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE static ssize_t dump_to_file(const char *path, const u8 *buf, size_t buf_len) { @@ -317,9 +427,9 @@ MODULE_PARM_DESC(rodata_dump_path, #endif /* WC_LINUXKM_SUPPORT_DUMP_TO_FILE */ #ifdef HAVE_FIPS - static ssize_t FIPS_rerun_self_test_handler(struct kobject *kobj, struct kobj_attribute *attr, + static ssize_t FIPS_rerun_self_test_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); - static struct kobj_attribute FIPS_rerun_self_test_attr = __ATTR(FIPS_rerun_self_test, 0220, NULL, FIPS_rerun_self_test_handler); + static struct module_attribute FIPS_rerun_self_test_attr = __ATTR(FIPS_rerun_self_test, 0220, NULL, FIPS_rerun_self_test_handler); static int installed_sysfs_FIPS_files = 0; #endif @@ -542,27 +652,37 @@ int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, byte* output, word32 sz) #include "linuxkm/x86_vector_register_glue.c" #endif -#ifdef CONFIG_HAVE_KPROBES - static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name); +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WC_C_DYNAMIC_FALLBACK) && \ + !defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) && \ + !defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) && \ + !defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) + #define WC_LINUXKM_SVR_DYNAMIC_AUDITING #endif #ifdef FIPS_OPTEST #ifndef HAVE_FIPS #error FIPS_OPTEST requires HAVE_FIPS. #endif - #ifdef LINUXKM_LKCAPI_REGISTER - #error FIPS_OPTEST is not allowed with LINUXKM_LKCAPI_REGISTER. - #endif extern int linuxkm_op_test_1(int argc, const char* argv[]); extern int linuxkm_op_test_wrapper(void); static wolfSSL_Atomic_Int *conTestFailure_ptr = NULL; #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT WOLFSSL_API extern wolfSSL_Atomic_Int wc_fips_optest_conTestFailure; #endif - static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attribute *attr, + static ssize_t FIPS_optest_trig_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); - static struct kobj_attribute FIPS_optest_trig_attr = __ATTR(FIPS_optest_run_code, 0220, NULL, FIPS_optest_trig_handler); + static struct module_attribute FIPS_optest_trig_attr = __ATTR(FIPS_optest_run_code, 0220, NULL, FIPS_optest_trig_handler); static int installed_sysfs_FIPS_optest_trig_files = 0; +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + static ssize_t FIPS_optest_trig_audit_accel_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, + const char *buf, size_t count); + static struct module_attribute FIPS_optest_trig_audit_accel_attr = __ATTR(FIPS_optest_run_code_audit_accel, 0220, NULL, FIPS_optest_trig_audit_accel_handler); + static int installed_sysfs_FIPS_optest_trig_audit_accel_files = 0; + static ssize_t FIPS_optest_trig_audit_c_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, + const char *buf, size_t count); + static struct module_attribute FIPS_optest_trig_audit_c_attr = __ATTR(FIPS_optest_run_code_audit_c, 0220, NULL, FIPS_optest_trig_audit_c_handler); + static int installed_sysfs_FIPS_optest_trig_audit_c_files = 0; +#endif #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) @@ -588,6 +708,14 @@ static int wolfssl_init(void) return ret; #endif +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(HAVE_FIPS) + ret = wc_linuxkm_allocate_svr_states(); + if (ret < 0) { + pr_err("wolfssl_init(): wc_linuxkm_allocate_svr_states() returned code %d.\n", ret); + return -ECANCELED; + } +#endif + #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE #ifdef WC_SYM_RELOC_TABLES @@ -666,6 +794,7 @@ static int wolfssl_init(void) #endif if (verifyCore_len != FIPS_IN_CORE_DIGEST_SIZE*2) { pr_err("ERROR: compile-time FIPS hash is the wrong length (expected %d hex digits, got %zu).\n", FIPS_IN_CORE_DIGEST_SIZE*2, verifyCore_len); + (void)libwolfssl_cleanup(); return -ECANCELED; } } @@ -677,6 +806,7 @@ static int wolfssl_init(void) ((uintptr_t)__wc_rodata_end < (uintptr_t)wolfCrypt_FIPS_ro_end)) { pr_err("ERROR: ELF segment fenceposts and FIPS fenceposts conflict.\n"); + (void)libwolfssl_cleanup(); return -ECANCELED; } #endif @@ -690,6 +820,7 @@ static int wolfssl_init(void) #ifdef CONFIG_MODULE_SIG if (THIS_MODULE->sig_ok == false) { pr_err("ERROR: wolfSSL module load aborted -- bad or missing module signature with FIPS dynamic hash.\n"); + (void)libwolfssl_cleanup(); return -ECANCELED; } #endif @@ -700,6 +831,7 @@ static int wolfssl_init(void) ret = updateFipsHash(); if (ret < 0) { pr_err("ERROR: wolfSSL module load aborted -- updateFipsHash: %s\n",wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); return -ECANCELED; } @@ -744,6 +876,7 @@ static int wolfssl_init(void) if (! canon_buf) { pr_err("ERROR: malloc(%d) for WOLFSSL_*_SEGMENT_CANONICALIZER failed.\n", WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ); + (void)libwolfssl_cleanup(); return -ECANCELED; } @@ -766,6 +899,7 @@ static int wolfssl_init(void) (unsigned)(uintptr_t)__wc_text_start, (unsigned)(uintptr_t)__wc_text_end); free(canon_buf); + (void)libwolfssl_cleanup(); return -ECANCELED; } stabilized_text_hash = hash_span(canon_buf, canon_buf + text_in_out_len, stabilized_text_hash); @@ -789,6 +923,7 @@ static int wolfssl_init(void) (unsigned)(uintptr_t)__wc_rodata_start, (unsigned)(uintptr_t)__wc_rodata_end); free(canon_buf); + (void)libwolfssl_cleanup(); return -ECANCELED; } stabilized_rodata_hash = hash_span(canon_buf, canon_buf + rodata_in_out_len, stabilized_rodata_hash); @@ -837,10 +972,11 @@ static int wolfssl_init(void) } #endif -#ifdef HAVE_FIPS +#if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) ret = wolfCrypt_SetCb_fips(lkmFipsCb); if (ret != 0) { pr_err("ERROR: wolfCrypt_SetCb_fips() failed: %s\n", wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); return -ECANCELED; } @@ -849,6 +985,14 @@ static int wolfssl_init(void) reloc_counts.other = 0; #endif + /* In asm builds, we run the FIPS self-test twice, once via fipsEntry() checking + * afterwards that no C fallbacks occurred, and a second time via + * wolfCrypt_IntegrityTest_fips() with asm disabled. + */ + +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + wc_svr_disallowed_count_reset(); +#endif if (WC_SIG_IGNORE_BEGIN() >= 0) { fipsEntry(); (void)WC_SIG_IGNORE_END(); @@ -876,9 +1020,67 @@ static int wolfssl_init(void) pr_err("ERROR: could not compute new hash. Contact customer support.\n"); } } + (void)libwolfssl_cleanup(); return -ECANCELED; } -#endif /* HAVE_FIPS */ + +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + { + long long unsigned int svr_disallowed_count = wc_svr_disallowed_count_current(); + #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) + long long unsigned int svr_disallowed_snapshot; + #endif + if (svr_disallowed_count > 0) { + pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after fipsEntry().\n", svr_disallowed_count); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + ret = DISABLE_VECTOR_REGISTERS(); + if (ret != 0) { + pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wolfCrypt_IntegrityTest_fips() returned %d.\n", ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) + /* DISABLE_VECTOR_REGISTERS() itself increments the disallowed + * count (it's a save call with WC_SVR_FLAG_INHIBIT), so snapshot + * after it and require the test run to increment past the + * snapshot -- a bare != 0 check after the test is vacuously + * satisfied by the DISABLE itself and detects nothing. */ + svr_disallowed_snapshot = wc_svr_disallowed_count_current(); + #endif + + ret = wolfCrypt_IntegrityTest_fips(); + + REENABLE_VECTOR_REGISTERS(); + + #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) + svr_disallowed_count = wc_svr_disallowed_count_current(); + if (svr_disallowed_count <= svr_disallowed_snapshot) { + pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wolfCrypt_IntegrityTest_fips() with DISABLE_VECTOR_REGISTERS() (snapshot %llu): inhibited-save instrumentation was not exercised.\n", svr_disallowed_count, svr_disallowed_snapshot); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + #endif + + if (ret != 0) { + pr_err("ERROR: wolfCrypt_IntegrityTest_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + ret = wolfCrypt_GetStatus_fips(); + if (ret != 0) { + pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + } +#endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ + +#endif /* HAVE_FIPS && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef WC_RNG_SEED_CB ret = wc_SetSeed_Cb(WC_GENERATE_SEED_DEFAULT); @@ -891,21 +1093,31 @@ static int wolfssl_init(void) } #endif /* WC_RNG_SEED_CB */ -#ifdef WOLFCRYPT_ONLY + /* Always call wolfCrypt_Init() directly, even in TLS builds, to assure + * mutex-free scheduler context during allocations. wolfSSL_Init() still + * uses a mutex, which puts the initialization thread in atomic context. + * The redundant wolfCrypt_Init() via wolfSSL_Init() is harmless -- + * wolfCrypt counts initialization depth. + */ ret = wolfCrypt_Init(); if (ret != 0) { pr_err("ERROR: wolfCrypt_Init() failed: %s\n", wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); return -ECANCELED; } -#else + wolfcrypt_inited = 1; + +#ifndef WOLFCRYPT_ONLY ret = wolfSSL_Init(); if (ret != WOLFSSL_SUCCESS) { pr_err("ERROR: wolfSSL_Init() failed: %s\n", wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); return -ECANCELED; } + wolfssl_inited = 1; #endif -#if defined(HAVE_FIPS) && FIPS_VERSION3_GT(5,2,0) +#if defined(HAVE_FIPS) && FIPS_VERSION3_GT(5,2,0) && !defined(WOLFSSL_FIPS_DEV_NO_POST) #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { @@ -915,6 +1127,10 @@ static int wolfssl_init(void) wc_linuxkm_stack_hwm_prepare(0xee); #endif +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + wc_svr_disallowed_count_reset(); +#endif + ret = wc_RunAllCast_fips(); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG @@ -927,9 +1143,72 @@ static int wolfssl_init(void) if (ret != 0) { pr_err("ERROR: wc_RunAllCast_fips() failed with return value %d\n", ret); + (void)libwolfssl_cleanup(); return -ECANCELED; } +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + { + long long unsigned int svr_disallowed_count = wc_svr_disallowed_count_current(); + long long unsigned int svr_disallowed_snapshot; + if (svr_disallowed_count > 0) { + pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips().\n", svr_disallowed_count); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + #ifdef WC_LINUXKM_HAVE_STACK_DEBUG + { + unsigned long stack_usage; + wc_linuxkm_stack_hwm_prepare(0xee); + #endif + + ret = DISABLE_VECTOR_REGISTERS(); + if (ret != 0) { + pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wc_RunAllCast_fips() returned %d.\n", ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + /* See the snapshot rationale in the wolfCrypt_IntegrityTest_fips() + * block above. */ + svr_disallowed_snapshot = wc_svr_disallowed_count_current(); + + ret = wc_RunAllCast_fips(); + + REENABLE_VECTOR_REGISTERS(); + + #ifdef WC_LINUXKM_HAVE_STACK_DEBUG + stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); + pr_info("STACK INFO: rel usage by wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS(): %lu\n", stack_usage); + /* shush up false stack HWM reading by kernel: */ + wc_linuxkm_stack_hwm_prepare(0); + } + #endif + + svr_disallowed_count = wc_svr_disallowed_count_current(); + if (svr_disallowed_count <= svr_disallowed_snapshot) { + pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() (snapshot %llu): inhibited-save instrumentation was not exercised.\n", svr_disallowed_count, svr_disallowed_snapshot); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + if (ret != 0) { + pr_err("ERROR: wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + + ret = wolfCrypt_GetStatus_fips(); + if (ret != 0) { + pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + } + +#endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ + pr_info("FIPS 140-3 wolfCrypt-fips v%d.%d.%d%s%s startup " "self-test succeeded.\n", #ifdef HAVE_FIPS_VERSION_MAJOR @@ -955,7 +1234,8 @@ static int wolfssl_init(void) "" #endif ); -#endif /* HAVE_FIPS && FIPS_VERSION3_GT(5,2,0) */ + +#endif /* HAVE_FIPS && FIPS_VERSION3_GT(5,2,0) && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef FIPS_OPTEST #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT @@ -964,15 +1244,32 @@ static int wolfssl_init(void) conTestFailure_ptr = (wolfSSL_Atomic_Int *)my_kallsyms_lookup_name("conTestFailure"); if (conTestFailure_ptr == NULL) { pr_err("ERROR: couldn't obtain conTestFailure_ptr.\n"); + (void)libwolfssl_cleanup(); return -ECANCELED; } #endif - ret = linuxkm_lkcapi_sysfs_install_node(&FIPS_optest_trig_attr, &installed_sysfs_FIPS_optest_trig_files); + ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_attr.attr, &installed_sysfs_FIPS_optest_trig_files); + if (ret != 0) { + pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_attr.attr.name, ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_audit_accel_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_accel_files); + if (ret != 0) { + pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_audit_accel_attr.attr.name, ret); + (void)libwolfssl_cleanup(); + return -ECANCELED; + } + ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_audit_c_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_c_files); if (ret != 0) { - pr_err("ERROR: linuxkm_lkcapi_sysfs_install_node() failed for %s (code %d).\n", FIPS_optest_trig_attr.attr.name, ret); + pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_audit_c_attr.attr.name, ret); + (void)libwolfssl_cleanup(); return -ECANCELED; } +#endif #ifdef FIPS_OPTEST_FULL_RUN_AT_MODULE_INIT @@ -1013,6 +1310,7 @@ static int wolfssl_init(void) ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { pr_err("ERROR: wolfCrypt_GetStatus_fips() after reset failed with code %d: %s\n", ret, wc_GetErrorString(ret)); + (void)libwolfssl_cleanup(); return -ECANCELED; } #endif @@ -1042,7 +1340,7 @@ static int wolfssl_init(void) } pr_info("wolfCrypt self-test passed.\n"); #else -#if !defined(HAVE_FIPS) || FIPS_VERSION3_LE(5,2,0) +#if !defined(HAVE_FIPS) || FIPS_VERSION3_LE(5,2,0) || defined(WOLFSSL_FIPS_DEV_NO_POST) pr_info("skipping full wolfcrypt_test() " "(configure with --enable-crypttests to enable).\n"); #endif @@ -1086,7 +1384,7 @@ static int wolfssl_init(void) #endif /* LINUXKM_LKCAPI_REGISTER */ #ifdef HAVE_FIPS - (void)linuxkm_lkcapi_sysfs_install_node(&FIPS_rerun_self_test_attr, &installed_sysfs_FIPS_files); + (void)linuxkm_sysfs_install_attr(&FIPS_rerun_self_test_attr.attr, &installed_sysfs_FIPS_files); #endif #ifdef WOLFSSL_LINUXKM_BENCHMARKS @@ -1141,11 +1439,13 @@ static void wolfssl_exit(void) #endif { #ifdef HAVE_FIPS - int ret; - - (void)linuxkm_lkcapi_sysfs_deinstall_node(&FIPS_rerun_self_test_attr, &installed_sysfs_FIPS_files); + (void)linuxkm_sysfs_deinstall_attr(&FIPS_rerun_self_test_attr.attr, &installed_sysfs_FIPS_files); #ifdef FIPS_OPTEST - (void)linuxkm_lkcapi_sysfs_deinstall_node(&FIPS_optest_trig_attr, &installed_sysfs_FIPS_optest_trig_files); + (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_attr.attr, &installed_sysfs_FIPS_optest_trig_files); +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_audit_accel_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_accel_files); + (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_audit_c_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_c_files); +#endif #endif #endif @@ -1154,13 +1454,15 @@ static void wolfssl_exit(void) (void)linuxkm_lkcapi_sysfs_deinstall(); #endif -#ifdef HAVE_FIPS - ret = wc_RunAllCast_fips(); - if (ret != 0) { - pr_err("ERROR: wc_RunAllCast_fips() failed at shutdown with return value %d\n", ret); +#if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) + { + int ret = wc_RunAllCast_fips(); + if (ret != 0) { + pr_err("ERROR: wc_RunAllCast_fips() failed at shutdown with return value %d\n", ret); + } + else + pr_info("wolfCrypt FIPS re-self-test succeeded at unload: all algorithms re-verified.\n"); } - else - pr_info("wolfCrypt FIPS re-self-test succeeded at unload: all algorithms re-verified.\n"); #endif (void)libwolfssl_cleanup(); @@ -1767,7 +2069,7 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { #endif /* WC_SYM_RELOC_TABLES */ -#if defined(HAVE_FIPS) && defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) +#if defined(HAVE_FIPS) && defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) && !defined(WOLFSSL_FIPS_DEV_NO_POST) #include @@ -1953,7 +2255,7 @@ static int updateFipsHash(void) return ret; } -#endif /* HAVE_FIPS && WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE */ +#endif /* HAVE_FIPS && WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef CONFIG_HAVE_KPROBES @@ -1995,13 +2297,13 @@ static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name) { #ifdef HAVE_FIPS -static ssize_t FIPS_rerun_self_test_handler(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t FIPS_rerun_self_test_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count) { int ret; - (void)kobj; - (void)attr; + (void)mattr; + (void)mk; /* only recognize "1" and "1\n". */ if ((count < 1) || (count > 2) || @@ -2041,6 +2343,30 @@ static ssize_t FIPS_rerun_self_test_handler(struct kobject *kobj, struct kobj_at return -EINVAL; } +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + + /* Note that wc_svr_disallowed_count*() can't be checked in + * FIPS_rerun_self_test_handler() -- we're already multiuser at this point + * and other threads can and will increment wc_svr_disallowed_count outside + * our control. + */ + + ret = DISABLE_VECTOR_REGISTERS(); + if (ret != 0) { + pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wc_RunAllCast_fips() returned %d.\n", ret); + return -EINVAL; + } + + ret = wc_RunAllCast_fips(); + + REENABLE_VECTOR_REGISTERS(); + + if (ret != 0) { + pr_err("ERROR: wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); + return -EINVAL; + } +#endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ + pr_info("wolfCrypt FIPS re-self-test succeeded: all algorithms verified and available.\n"); return count; @@ -2052,7 +2378,13 @@ typedef struct test_func_args { int return_code; } test_func_args; -static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attribute *attr, +enum FIPS_optest_audit_mode { + FIPS_OPTEST_AUDIT_NONE = 0, + FIPS_OPTEST_AUDIT_ACCEL, + FIPS_OPTEST_AUDIT_C +}; + +static ssize_t FIPS_optest_trig_common(enum FIPS_optest_audit_mode audit_mode, const char *buf, const size_t count) { int ret; @@ -2061,9 +2393,13 @@ static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attrib char code_buf[5]; size_t corrected_count; int i; - - (void)kobj; - (void)attr; + static wolfSSL_Atomic_Int in_optest = WOLFSSL_ATOMIC_INITIALIZER(-1); +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + long long unsigned int svr_disallowed_before_optest = 0, svr_disallowed_after_optest; + ssize_t ret_count = 0; +#else + (void)audit_mode; +#endif /* buf may or may not have an LF at end -- tolerate both. there is no * terminating null in either case. @@ -2087,6 +2423,40 @@ static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attrib argv[2] = code_buf; argc = 3; +#ifdef LINUXKM_LKCAPI_REGISTER + { + int current_linuxkm_lkcapi_registering_now = 0; + + if (! wolfSSL_Atomic_Int_CompareExchange( + &linuxkm_lkcapi_registering_now, + ¤t_linuxkm_lkcapi_registering_now, + 1)) { + pr_err("ERROR: FIPS_optest_trig_handler() with linuxkm_lkcapi_registered: before running optest, echo 1 > /sys/module/libwolfssl/deinstall_algs.\n"); + return -EBUSY; + } + + if (linuxkm_lkcapi_registered) { + WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); + pr_err("ERROR: FIPS_optest_trig_handler() with linuxkm_lkcapi_registered: before running optest, echo 1 > /sys/module/libwolfssl/deinstall_algs.\n"); + return -EBUSY; + } + } +#endif + + { + WC_ATOMIC_INT_ARG expected_in_optest = -1; + if (! wolfSSL_Atomic_Int_CompareExchange( + &in_optest, &expected_in_optest, task_pid_nr(current))) + { +#ifdef LINUXKM_LKCAPI_REGISTER + WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); +#endif + pr_err("ERROR: FIPS_optest_trig_handler() called by pid %d while pid %d is running optest in another thread.\n", + task_pid_nr(current), expected_in_optest); + return -EBUSY; + } + } + printf("OK, testing code %s\n", code_buf); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG @@ -2097,8 +2467,50 @@ static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attrib wc_linuxkm_stack_hwm_prepare(0xee); #endif +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + + /* The audits treat wc_svr_disallowed_count as + * ours alone: in_optest excludes the other trigger nodes, and + * getting linuxkm_lkcapi_registering_now then checking + * linuxkm_lkcapi_registered excludes registered- + * algorithm consumers. + */ + + if ((audit_mode == FIPS_OPTEST_AUDIT_ACCEL) || + (audit_mode == FIPS_OPTEST_AUDIT_C)) + { + if (audit_mode == FIPS_OPTEST_AUDIT_C) { + ret = DISABLE_VECTOR_REGISTERS(); + if (ret != 0) { + pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for FIPS_optest_trig_handler() returned %d.\n", ret); + ret_count = -EINVAL; + goto out; + } + } + svr_disallowed_before_optest = wc_svr_disallowed_count_current(); + } +#endif + ret = linuxkm_op_test_1(argc, &argv[0]); +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + if (audit_mode == FIPS_OPTEST_AUDIT_C) { + REENABLE_VECTOR_REGISTERS(); + svr_disallowed_after_optest = wc_svr_disallowed_count_current(); + if (svr_disallowed_after_optest == svr_disallowed_before_optest) { + pr_err("ERROR: wc_svr_disallowed_count_current() did not increment during optest with DISABLE_VECTOR_REGISTERS().\n"); + ret_count = -EINVAL; + } + } + else if (audit_mode == FIPS_OPTEST_AUDIT_ACCEL) { + svr_disallowed_after_optest = wc_svr_disallowed_count_current(); + if (svr_disallowed_after_optest != svr_disallowed_before_optest) { + pr_err("ERROR: wc_svr_disallowed_count_current() incremented (+%llu) during optest.\n", svr_disallowed_after_optest - svr_disallowed_before_optest); + ret_count = -EINVAL; + } + } +#endif + #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by linuxkm_op_test_1(): %lu\n", stack_usage); @@ -2109,6 +2521,10 @@ static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attrib printf("ret of op_test = %d\n", ret); +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING +out: +#endif + /* reload the library in memory and re-init state */ printf("Reloading the module in memory (equivalent to power " "cycle)\n"); @@ -2131,9 +2547,46 @@ static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attrib printf("Module status is: %d\n", wolfCrypt_GetStatus_fips()); printf("Module mode is: %d\n", wolfCrypt_GetMode_fips()); - return count; + WOLFSSL_ATOMIC_STORE(in_optest, -1); +#ifdef LINUXKM_LKCAPI_REGISTER + WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); +#endif + +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + return ret_count ? ret_count : (ssize_t)count; +#else + return (ssize_t)count; +#endif } +static ssize_t FIPS_optest_trig_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, + const char *buf, const size_t count) +{ + (void)mattr; + (void)mk; + return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_NONE, buf, count); +} + +#ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING + +static ssize_t FIPS_optest_trig_audit_accel_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, + const char *buf, const size_t count) +{ + (void)mattr; + (void)mk; + return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_ACCEL, buf, count); +} + +static ssize_t FIPS_optest_trig_audit_c_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, + const char *buf, const size_t count) +{ + (void)mattr; + (void)mk; + return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_C, buf, count); +} + +#endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ + #endif /* FIPS_OPTEST */ #endif /* HAVE_FIPS */ diff --git a/wolfssl/wolfcrypt/error-crypt.h b/wolfssl/wolfcrypt/error-crypt.h index 32f80499030..fad6f926c7c 100644 --- a/wolfssl/wolfcrypt/error-crypt.h +++ b/wolfssl/wolfcrypt/error-crypt.h @@ -383,9 +383,12 @@ WOLFSSL_ABI WOLFSSL_API const char* wc_GetErrorString(int error); WOLFSSL_API extern int wc_backtrace_render(void); #endif -#if defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES) && \ - (defined(BUILDING_WOLFSSL) || \ - defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES_ALWAYS)) +#if defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES) && \ + (defined(BUILDING_WOLFSSL) || \ + defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES_ALWAYS)) && \ + (!defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES_PER_FILE) || \ + defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES_THIS_FILE)) + #include #define WC_NO_ERR_TRACE(label) (CONST_NUM_ERR_ ## label) #ifndef WOLFSSL_DEBUG_BACKTRACE_RENDER_CLAUSE diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index bdaa0f2117c..0d400713438 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -579,15 +579,16 @@ #define WOLFSSL_ATOMIC_LOAD(x) atomic_load(&(x)) #define WOLFSSL_ATOMIC_STORE(x, val) atomic_store(&(x), val) #define WOLFSSL_ATOMIC_OPS - #elif defined(__GNUC__) && defined(__ATOMIC_CONSUME) - /* direct calls using gcc-style compiler built-ins */ + #elif defined(__GNUC__) && defined(__ATOMIC_SEQ_CST) + /* direct calls using gcc-style compiler built-ins, matching the + * implicit semantics of atomic_load() and atomic_store() */ typedef volatile int wolfSSL_Atomic_Int; typedef volatile unsigned int wolfSSL_Atomic_Uint; #define WOLFSSL_ATOMIC_INITIALIZER(x) (x) #define WOLFSSL_ATOMIC_LOAD(x) __atomic_load_n(&(x), \ - __ATOMIC_CONSUME) + __ATOMIC_SEQ_CST) #define WOLFSSL_ATOMIC_STORE(x, val) __atomic_store_n(&(x), \ - val, __ATOMIC_RELEASE) + val, __ATOMIC_SEQ_CST) #define WOLFSSL_ATOMIC_OPS #elif defined(_MSC_VER) && defined(USE_WINDOWS_API) && \ !defined(WOLFSSL_NOT_WINDOWS_API)