@@ -10569,6 +10569,13 @@ F: Documentation/litmus-tests/
F: Documentation/memory-barriers.txt
F: tools/memory-model/
+LINUX RANDOM NUMBER GENERATOR (LRNG) DRIVER
+M: Stephan Mueller <smueller@chronox.de>
+S: Maintained
+W: https://www.chronox.de/lrng.html
+F: drivers/char/lrng/*
+F: include/linux/lrng.h
+
LIS3LV02D ACCELEROMETER DRIVER
M: Eric Piel <eric.piel@tremplin-utc.net>
S: Maintained
@@ -461,6 +461,8 @@ config ADI
and SSM (Silicon Secured Memory). Intended consumers of this
driver include crash and makedumpfile.
+source "drivers/char/lrng/Kconfig"
+
endmenu
config RANDOM_TRUST_CPU
@@ -3,7 +3,14 @@
# Makefile for the kernel character device drivers.
#
-obj-y += mem.o random.o
+obj-y += mem.o
+
+ifeq ($(CONFIG_LRNG),y)
+ obj-y += lrng/
+else
+ obj-y += random.o
+endif
+
obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o
obj-y += misc.o
obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o
new file mode 100644
@@ -0,0 +1,205 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Linux Random Number Generator configuration
+#
+
+menuconfig LRNG
+ bool "Linux Random Number Generator"
+ select CRYPTO_LIB_SHA256 if CRYPTO
+ help
+ The Linux Random Number Generator (LRNG) is the replacement
+ of the existing /dev/random provided with drivers/char/random.c.
+ It generates entropy from different noise sources and
+ delivers significant entropy during boot.
+
+if LRNG
+
+menu "Specific DRNG seeding strategies"
+
+config LRNG_OVERSAMPLE_ENTROPY_SOURCES
+ bool "Oversample entropy sources"
+ default n
+ help
+ When enabling this option, the entropy sources are
+ over-sampled with the following approach: First, the
+ the entropy sources are requested to provide 64 bits more
+ entropy than the size of the entropy buffer. For example,
+ if the entropy buffer is 256 bits, 320 bits of entropy
+ is requested to fill that buffer.
+
+ Second, the seed operation of the deterministic RNG
+ requests 128 bits more data from each entropy source than
+ the security strength of the DRNG during initialization.
+ A prerequisite for this operation is that the digest size
+ of the used hash must be at least equally large to generate
+ that buffer. If the prerequisite is not met, this
+ oversampling is not applied.
+
+ This strategy is intended to offset the asymptotic entropy
+ increase to reach full entropy in a buffer.
+
+ The strategy is consistent with the requirements in
+ NIST SP800-90C and is only enforced with fips=1.
+
+ If unsure, say N.
+
+config LRNG_OVERSAMPLE_ES_BITS
+ int
+ default 0 if !LRNG_OVERSAMPLE_ENTROPY_SOURCES
+ default 64 if LRNG_OVERSAMPLE_ENTROPY_SOURCES
+
+config LRNG_SEED_BUFFER_INIT_ADD_BITS
+ int
+ default 0 if !LRNG_OVERSAMPLE_ENTROPY_SOURCES
+ default 128 if LRNG_OVERSAMPLE_ENTROPY_SOURCES
+
+endmenu # "Specific DRNG seeding strategies"
+
+menu "Entropy Source Configuration"
+
+comment "Interrupt Entropy Source"
+
+choice
+ prompt "Continuous entropy compression boot time setting"
+ default LRNG_CONTINUOUS_COMPRESSION_ENABLED
+ help
+ Select the default behavior of the interrupt entropy source
+ continuous compression operation.
+
+ The Linux RNG collects entropy data during each interrupt.
+ For performance reasons, a amount of entropy data defined by
+ the LRNG entropy collection pool size is concatenated into
+ an array. When that array is filled up, a hash is calculated
+ to compress the entropy. That hash is calculated in
+ interrupt context.
+
+ In case such hash calculation in interrupt context is deemed
+ too time-consuming, the continuous compression operation
+ can be disabled. If disabled, the collection of entropy will
+ not trigger a hash compression operation in interrupt context.
+ The compression happens only when the DRNG is reseeded which is
+ in process context. This implies that old entropy data
+ collected after the last DRNG-reseed is overwritten with newer
+ entropy data once the collection pool is full instead of
+ retaining its entropy with the compression operation.
+
+ config LRNG_CONTINUOUS_COMPRESSION_ENABLED
+ bool "Enable continuous compression (default)"
+
+ config LRNG_CONTINUOUS_COMPRESSION_DISABLED
+ bool "Disable continuous compression"
+endchoice
+
+config LRNG_ENABLE_CONTINUOUS_COMPRESSION
+ bool
+ default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED
+ default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED
+
+config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION
+ bool "Runtime-switchable continuous entropy compression"
+ help
+ Per default, the interrupt entropy source continuous
+ compression operation behavior is hard-wired into the kernel.
+ Enable this option to allow it to be configurable at boot time.
+
+ To modify the default behavior of the continuous
+ compression operation, use the kernel command line option
+ of lrng_sw_noise.lrng_pcpu_continuous_compression.
+
+ If unsure, say N.
+
+choice
+ prompt "LRNG Entropy Collection Pool Size"
+ default LRNG_COLLECTION_SIZE_1024
+ help
+ Select the size of the LRNG entropy collection pool
+ storing data for the interrupt entropy source without
+ performing a compression operation. The larger the
+ collection size is, the faster the average interrupt
+ handling will be. The collection size represents the
+ number of bytes of the per-CPU memory used to batch
+ up entropy event data.
+
+ The default value is good for regular operations. Choose
+ larger sizes for servers that have no memory limitations.
+ If runtime memory is precious, choose a smaller size.
+
+ The collection size is unrelated to the entropy rate
+ or the amount of entropy the LRNG can process.
+
+ config LRNG_COLLECTION_SIZE_32
+ depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED
+ depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION
+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES
+ bool "32 interrupt events"
+
+ config LRNG_COLLECTION_SIZE_256
+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES
+ bool "256 interrupt events"
+
+ config LRNG_COLLECTION_SIZE_512
+ bool "512 interrupt events"
+
+ config LRNG_COLLECTION_SIZE_1024
+ bool "1024 interrupt events (default)"
+
+ config LRNG_COLLECTION_SIZE_2048
+ bool "2048 interrupt events"
+
+ config LRNG_COLLECTION_SIZE_4096
+ bool "4096 interrupt events"
+
+ config LRNG_COLLECTION_SIZE_8192
+ bool "8192 interrupt events"
+
+endchoice
+
+config LRNG_COLLECTION_SIZE
+ int
+ default 32 if LRNG_COLLECTION_SIZE_32
+ default 256 if LRNG_COLLECTION_SIZE_256
+ default 512 if LRNG_COLLECTION_SIZE_512
+ default 1024 if LRNG_COLLECTION_SIZE_1024
+ default 2048 if LRNG_COLLECTION_SIZE_2048
+ default 4096 if LRNG_COLLECTION_SIZE_4096
+ default 8192 if LRNG_COLLECTION_SIZE_8192
+
+config LRNG_IRQ_ENTROPY_RATE
+ int "Interrupt Entropy Source Entropy Rate"
+ range 256 4294967295
+ default 256
+ help
+ The LRNG will collect the configured number of interrupts to
+ obtain 256 bits of entropy. This value can be set to any between
+ 256 and 4294967295. The LRNG guarantees that this value is not
+ lower than 256. This lower limit implies that one interrupt event
+ is credited with one bit of entropy. This value is subject to the
+ increase by the oversampling factor, if no high-resolution timer
+ is found.
+
+ In order to effectively disable the interrupt entropy source,
+ the option has to be set to 4294967295. In this case, the
+ interrupt entropy source will still deliver data but without
+ being credited with entropy.
+
+comment "CPU Entropy Source"
+
+config LRNG_CPU_ENTROPY_RATE
+ int "CPU Entropy Source Entropy Rate"
+ range 0 256
+ default 8
+ help
+ The option defines the amount of entropy the LRNG applies to 256
+ bits of data obtained from the CPU entropy source. The LRNG
+ enforces the limit that this value must be in the range between
+ 0 and 256.
+
+ In order to disable the CPU entropy source, the option has to
+ be set to 0.
+
+ Note, this option is overwritten when the option
+ CONFIG_RANDOM_TRUST_CPU is set.
+
+endmenu # "Entropy Source Configuration"
+
+endif # LRNG
new file mode 100644
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the Linux Random Number Generator.
+#
+
+obj-y += lrng_pool.o lrng_aux.o \
+ lrng_sw_noise.o lrng_archrandom.o \
+ lrng_drng.o lrng_chacha20.o \
+ lrng_interfaces.o
new file mode 100644
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG Fast Entropy Source: CPU-based entropy source
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/random.h>
+
+#include "lrng_internal.h"
+
+/*
+ * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS.
+ * As we have no ability to review the implementation of those noise sources,
+ * it is prudent to have a conservative estimate here.
+ */
+#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE
+#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS
+#ifdef CONFIG_RANDOM_TRUST_CPU
+static u32 archrandom = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH;
+#else
+static u32 archrandom = LRNG_ARCHRANDOM_DEFAULT_STRENGTH;
+#endif
+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG
+module_param(archrandom, uint, 0644);
+MODULE_PARM_DESC(archrandom, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDRAND)");
+#endif
+
+static int __init lrng_parse_trust_cpu(char *arg)
+{
+ int ret;
+ bool trust_cpu = false;
+
+ ret = kstrtobool(arg, &trust_cpu);
+ if (ret)
+ return ret;
+
+ if (trust_cpu) {
+ archrandom = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH;
+ lrng_pool_add_entropy();
+ } else {
+ archrandom = LRNG_ARCHRANDOM_DEFAULT_STRENGTH;
+ }
+
+ return 0;
+}
+early_param("random.trust_cpu", lrng_parse_trust_cpu);
+
+u32 lrng_archrandom_entropylevel(u32 requested_bits)
+{
+ return lrng_fast_noise_entropylevel(archrandom, requested_bits);
+}
+
+/**
+ * lrng_get_arch() - Get CPU noise source entropy
+ *
+ * @outbuf: buffer to store entropy of size LRNG_DRNG_SECURITY_STRENGTH_BYTES
+ *
+ * Return:
+ * * > 0 on success where value provides the added entropy in bits
+ * * 0 if no fast source was available
+ */
+u32 lrng_get_arch(u8 *outbuf, u32 requested_bits)
+{
+ u32 i, ent_bits = lrng_archrandom_entropylevel(requested_bits);
+
+ /* operate on full blocks */
+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long));
+ BUILD_BUG_ON(CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS %
+ sizeof(unsigned long));
+ /* ensure we have aligned buffers */
+ BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long));
+
+ if (!ent_bits)
+ return 0;
+
+ for (i = 0; i < (requested_bits >> 3);
+ i += sizeof(unsigned long)) {
+ if (!arch_get_random_seed_long((unsigned long *)(outbuf + i)) &&
+ !arch_get_random_long((unsigned long *)(outbuf + i))) {
+ archrandom = 0;
+ return 0;
+ }
+ }
+
+ pr_debug("obtained %u bits of entropy from CPU RNG noise source\n",
+ ent_bits);
+ return ent_bits;
+}
new file mode 100644
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG auxiliary interfaces
+ *
+ * Copyright (C) 2019 - 2021 Stephan Mueller <smueller@chronox.de>
+ * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All
+ * Rights Reserved.
+ * Copyright (C) 2016 Jason Cooper <jason@lakedaemon.net>
+ */
+
+#include <linux/mm.h>
+#include <linux/random.h>
+
+#include "lrng_internal.h"
+
+struct batched_entropy {
+ union {
+ u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)];
+ u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)];
+ };
+ unsigned int position;
+ spinlock_t batch_lock;
+};
+
+/*
+ * Get a random word for internal kernel use only. The quality of the random
+ * number is as good as /dev/urandom, but there is no backtrack protection,
+ * with the goal of being quite fast and not depleting entropy.
+ */
+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
+};
+
+u64 get_random_u64(void)
+{
+ u64 ret;
+ unsigned long flags;
+ struct batched_entropy *batch;
+
+ lrng_debug_report_seedlevel("get_random_u64");
+
+ batch = raw_cpu_ptr(&batched_entropy_u64);
+ spin_lock_irqsave(&batch->batch_lock, flags);
+ if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
+ lrng_drng_get_atomic((u8 *)batch->entropy_u64,
+ LRNG_DRNG_BLOCKSIZE);
+ batch->position = 0;
+ }
+ ret = batch->entropy_u64[batch->position++];
+ spin_unlock_irqrestore(&batch->batch_lock, flags);
+ return ret;
+}
+EXPORT_SYMBOL(get_random_u64);
+
+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
+};
+
+u32 get_random_u32(void)
+{
+ u32 ret;
+ unsigned long flags;
+ struct batched_entropy *batch;
+
+ lrng_debug_report_seedlevel("get_random_u32");
+
+ batch = raw_cpu_ptr(&batched_entropy_u32);
+ spin_lock_irqsave(&batch->batch_lock, flags);
+ if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
+ lrng_drng_get_atomic((u8 *)batch->entropy_u32,
+ LRNG_DRNG_BLOCKSIZE);
+ batch->position = 0;
+ }
+ ret = batch->entropy_u32[batch->position++];
+ spin_unlock_irqrestore(&batch->batch_lock, flags);
+ return ret;
+}
+EXPORT_SYMBOL(get_random_u32);
+
+/*
+ * It's important to invalidate all potential batched entropy that might
+ * be stored before the crng is initialized, which we can do lazily by
+ * simply resetting the counter to zero so that it's re-extracted on the
+ * next usage.
+ */
+void invalidate_batched_entropy(void)
+{
+ int cpu;
+ unsigned long flags;
+
+ for_each_possible_cpu(cpu) {
+ struct batched_entropy *batched_entropy;
+
+ batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
+ spin_lock_irqsave(&batched_entropy->batch_lock, flags);
+ batched_entropy->position = 0;
+ spin_unlock(&batched_entropy->batch_lock);
+
+ batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
+ spin_lock(&batched_entropy->batch_lock);
+ batched_entropy->position = 0;
+ spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
+ }
+}
+
+/**
+ * randomize_page - Generate a random, page aligned address
+ * @start: The smallest acceptable address the caller will take.
+ * @range: The size of the area, starting at @start, within which the
+ * random address must fall.
+ *
+ * If @start + @range would overflow, @range is capped.
+ *
+ * NOTE: Historical use of randomize_range, which this replaces, presumed that
+ * @start was already page aligned. We now align it regardless.
+ *
+ * Return: A page aligned address within [start, start + range). On error,
+ * @start is returned.
+ */
+unsigned long randomize_page(unsigned long start, unsigned long range)
+{
+ if (!PAGE_ALIGNED(start)) {
+ range -= PAGE_ALIGN(start) - start;
+ start = PAGE_ALIGN(start);
+ }
+
+ if (start > ULONG_MAX - range)
+ range = ULONG_MAX - start;
+
+ range >>= PAGE_SHIFT;
+
+ if (range == 0)
+ return start;
+
+ return start + (get_random_long() % range << PAGE_SHIFT);
+}
new file mode 100644
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Backend for the LRNG providing the cryptographic primitives using
+ * ChaCha20 cipher implementations.
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <crypto/chacha.h>
+#include <linux/lrng.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+
+#include "lrng_chacha20.h"
+#include "lrng_internal.h"
+
+/******************************* ChaCha20 DRNG *******************************/
+
+#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32))
+
+struct chacha20_state {
+ struct chacha20_block block;
+};
+
+/*
+ * Have a static memory blocks for the ChaCha20 DRNG instance to avoid calling
+ * kmalloc too early in the boot cycle. For subsequent allocation requests,
+ * such as per-NUMA-node DRNG instances, kmalloc will be used.
+ */
+struct chacha20_state chacha20 __latent_entropy;
+
+/**
+ * Update of the ChaCha20 state by either using an unused buffer part or by
+ * generating one ChaCha20 block which is half of the state of the ChaCha20.
+ * The block is XORed into the key part of the state. This shall ensure
+ * backtracking resistance as well as a proper mix of the ChaCha20 state once
+ * the key is injected.
+ */
+static void lrng_chacha20_update(struct chacha20_state *chacha20_state,
+ __le32 *buf, u32 used_words)
+{
+ struct chacha20_block *chacha20 = &chacha20_state->block;
+ u32 i;
+ __le32 tmp[CHACHA_BLOCK_WORDS];
+
+ BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE);
+ BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE);
+
+ if (used_words > CHACHA_KEY_SIZE_WORDS) {
+ chacha20_block(&chacha20->constants[0], (u8 *)tmp);
+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++)
+ chacha20->key.u[i] ^= le32_to_cpu(tmp[i]);
+ memzero_explicit(tmp, sizeof(tmp));
+ } else {
+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++)
+ chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]);
+ }
+
+ /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */
+ chacha20->nonce[0]++;
+ if (chacha20->nonce[0] == 0) {
+ chacha20->nonce[1]++;
+ if (chacha20->nonce[1] == 0)
+ chacha20->nonce[2]++;
+ }
+
+ /* Leave counter untouched as it is start value is undefined in RFC */
+}
+
+/*
+ * Seed the ChaCha20 DRNG by injecting the input data into the key part of
+ * the ChaCha20 state. If the input data is longer than the ChaCha20 key size,
+ * perform a ChaCha20 operation after processing of key size input data.
+ * This operation shall spread out the entropy into the ChaCha20 state before
+ * new entropy is injected into the key part.
+ */
+static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen)
+{
+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng;
+ struct chacha20_block *chacha20 = &chacha20_state->block;
+
+ while (inbuflen) {
+ u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE);
+
+ for (i = 0; i < todo; i++)
+ chacha20->key.b[i] ^= inbuf[i];
+
+ /* Break potential dependencies between the inbuf key blocks */
+ lrng_chacha20_update(chacha20_state, NULL,
+ CHACHA_BLOCK_WORDS);
+ inbuf += todo;
+ inbuflen -= todo;
+ }
+
+ return 0;
+}
+
+/*
+ * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20
+ * is the random number. After the completion of the generation of the
+ * stream, the entire ChaCha20 state is updated.
+ *
+ * Note, as the ChaCha20 implements a 32 bit counter, we must ensure
+ * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks
+ * before a reseed or an update happens. This is ensured by the variable
+ * outbuflen which is a 32 bit integer defining the number of bytes to be
+ * generated by the ChaCha20 DRNG. At the end of this function, an update
+ * operation is invoked which implies that the 32 bit counter will never be
+ * overflown in this implementation.
+ */
+static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen)
+{
+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng;
+ struct chacha20_block *chacha20 = &chacha20_state->block;
+ __le32 aligned_buf[CHACHA_BLOCK_WORDS];
+ u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS;
+ int zeroize_buf = 0;
+
+ while (outbuflen >= CHACHA_BLOCK_SIZE) {
+ chacha20_block(&chacha20->constants[0], outbuf);
+ outbuf += CHACHA_BLOCK_SIZE;
+ outbuflen -= CHACHA_BLOCK_SIZE;
+ }
+
+ if (outbuflen) {
+ chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf);
+ memcpy(outbuf, aligned_buf, outbuflen);
+ used = ((outbuflen + sizeof(aligned_buf[0]) - 1) /
+ sizeof(aligned_buf[0]));
+ zeroize_buf = 1;
+ }
+
+ lrng_chacha20_update(chacha20_state, aligned_buf, used);
+
+ if (zeroize_buf)
+ memzero_explicit(aligned_buf, sizeof(aligned_buf));
+
+ return ret;
+}
+
+void lrng_cc20_init_state(struct chacha20_state *state)
+{
+ lrng_cc20_init_rfc7539(&state->block);
+}
+
+/*
+ * Allocation of the DRNG state
+ */
+static void *lrng_cc20_drng_alloc(u32 sec_strength)
+{
+ struct chacha20_state *state = NULL;
+
+ if (sec_strength > CHACHA_KEY_SIZE) {
+ pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n",
+ CHACHA_KEY_SIZE * 8, sec_strength * 8);
+ return ERR_PTR(-EINVAL);
+ }
+ if (sec_strength < CHACHA_KEY_SIZE)
+ pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n",
+ CHACHA_KEY_SIZE * 8, sec_strength * 8);
+
+ state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL);
+ if (!state)
+ return ERR_PTR(-ENOMEM);
+ pr_debug("memory for ChaCha20 core allocated\n");
+
+ lrng_cc20_init_state(state);
+
+ return state;
+}
+
+static void lrng_cc20_drng_dealloc(void *drng)
+{
+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng;
+
+ if (drng == &chacha20) {
+ memzero_explicit(chacha20_state, sizeof(*chacha20_state));
+ pr_debug("static ChaCha20 core zeroized\n");
+ return;
+ }
+
+ pr_debug("ChaCha20 core zeroized and freed\n");
+ kfree_sensitive(chacha20_state);
+}
+
+/******************************* Hash Operation *******************************/
+
+#ifdef CONFIG_CRYPTO_LIB_SHA256
+
+#include <crypto/sha2.h>
+
+static u32 lrng_cc20_hash_digestsize(void *hash)
+{
+ return SHA256_DIGEST_SIZE;
+}
+
+static int lrng_cc20_hash_init(struct shash_desc *shash, void *hash)
+{
+ /*
+ * We do not need a TFM - we only need sufficient space for
+ * struct sha256_state on the stack.
+ */
+ sha256_init(shash_desc_ctx(shash));
+ return 0;
+}
+
+static int lrng_cc20_hash_update(struct shash_desc *shash,
+ const u8 *inbuf, u32 inbuflen)
+{
+ sha256_update(shash_desc_ctx(shash), inbuf, inbuflen);
+ return 0;
+}
+
+static int lrng_cc20_hash_final(struct shash_desc *shash, u8 *digest)
+{
+ sha256_final(shash_desc_ctx(shash), digest);
+ return 0;
+}
+
+static const char *lrng_cc20_hash_name(void)
+{
+ return "SHA-256";
+}
+
+static void lrng_cc20_hash_desc_zero(struct shash_desc *shash)
+{
+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state));
+}
+
+#else /* CONFIG_CRYPTO_LIB_SHA256 */
+
+#include <crypto/sha1.h>
+#include <crypto/sha1_base.h>
+
+/*
+ * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always
+ * compiled and present in the kernel.
+ */
+static u32 lrng_cc20_hash_digestsize(void *hash)
+{
+ return SHA1_DIGEST_SIZE;
+}
+
+static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src,
+ int blocks)
+{
+ u32 temp[SHA1_WORKSPACE_WORDS];
+
+ while (blocks--) {
+ sha1_transform(sctx->state, src, temp);
+ src += SHA1_BLOCK_SIZE;
+ }
+ memzero_explicit(temp, sizeof(temp));
+}
+
+static int lrng_cc20_hash_init(struct shash_desc *shash, void *hash)
+{
+ /*
+ * We do not need a TFM - we only need sufficient space for
+ * struct sha1_state on the stack.
+ */
+ sha1_base_init(shash);
+ return 0;
+}
+
+static int lrng_cc20_hash_update(struct shash_desc *shash,
+ const u8 *inbuf, u32 inbuflen)
+{
+ return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn);
+}
+
+static int lrng_cc20_hash_final(struct shash_desc *shash, u8 *digest)
+{
+ return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?:
+ sha1_base_finish(shash, digest);
+}
+
+static const char *lrng_cc20_hash_name(void)
+{
+ return "SHA-1";
+}
+
+static void lrng_cc20_hash_desc_zero(struct shash_desc *shash)
+{
+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state));
+}
+
+#endif /* CONFIG_CRYPTO_LIB_SHA256 */
+
+static void *lrng_cc20_hash_alloc(void)
+{
+ pr_info("Hash %s allocated\n", lrng_cc20_hash_name());
+ return NULL;
+}
+
+static void lrng_cc20_hash_dealloc(void *hash)
+{
+}
+
+static const char *lrng_cc20_drng_name(void)
+{
+ return "ChaCha20 DRNG";
+}
+
+const struct lrng_crypto_cb lrng_cc20_crypto_cb = {
+ .lrng_drng_name = lrng_cc20_drng_name,
+ .lrng_hash_name = lrng_cc20_hash_name,
+ .lrng_drng_alloc = lrng_cc20_drng_alloc,
+ .lrng_drng_dealloc = lrng_cc20_drng_dealloc,
+ .lrng_drng_seed_helper = lrng_cc20_drng_seed_helper,
+ .lrng_drng_generate_helper = lrng_cc20_drng_generate_helper,
+ .lrng_hash_alloc = lrng_cc20_hash_alloc,
+ .lrng_hash_dealloc = lrng_cc20_hash_dealloc,
+ .lrng_hash_digestsize = lrng_cc20_hash_digestsize,
+ .lrng_hash_init = lrng_cc20_hash_init,
+ .lrng_hash_update = lrng_cc20_hash_update,
+ .lrng_hash_final = lrng_cc20_hash_final,
+ .lrng_hash_desc_zero = lrng_cc20_hash_desc_zero,
+};
new file mode 100644
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * LRNG ChaCha20 definitions
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#include <crypto/chacha.h>
+
+/* State according to RFC 7539 section 2.3 */
+struct chacha20_block {
+ u32 constants[4];
+ union {
+#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
+ u32 u[CHACHA_KEY_SIZE_WORDS];
+ u8 b[CHACHA_KEY_SIZE];
+ } key;
+ u32 counter;
+ u32 nonce[3];
+};
+
+static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20)
+{
+ /* String "expand 32-byte k" */
+ chacha20->constants[0] = 0x61707865;
+ chacha20->constants[1] = 0x3320646e;
+ chacha20->constants[2] = 0x79622d32;
+ chacha20->constants[3] = 0x6b206574;
+}
new file mode 100644
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG DRNG processing
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/fips.h>
+#include <linux/lrng.h>
+
+#include "lrng_internal.h"
+
+/*
+ * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note,
+ * this is enforced with the next request of random numbers from the
+ * DRNG. Setting this value to zero implies a reseeding attempt before every
+ * generated random number.
+ */
+int lrng_drng_reseed_max_time = 600;
+
+static atomic_t lrng_avail = ATOMIC_INIT(0);
+
+DEFINE_MUTEX(lrng_crypto_cb_update);
+
+/* DRNG for /dev/urandom, getrandom(2), get_random_bytes */
+static struct lrng_drng lrng_drng_init = {
+ .drng = &chacha20,
+ .crypto_cb = &lrng_cc20_crypto_cb,
+ .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock),
+ .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_init.spin_lock),
+ .hash_lock = __RW_LOCK_UNLOCKED(lrng_drng_init.hash_lock)
+};
+
+/*
+ * DRNG for get_random_bytes when called in atomic context. This
+ * DRNG will always use the ChaCha20 DRNG. It will never benefit from a
+ * DRNG switch like the "regular" DRNG. If there was no DRNG switch, the atomic
+ * DRNG is identical to the "regular" DRNG.
+ *
+ * The reason for having this is due to the fact that DRNGs other than
+ * the ChaCha20 DRNG may sleep.
+ */
+static struct lrng_drng lrng_drng_atomic = {
+ .drng = &chacha20,
+ .crypto_cb = &lrng_cc20_crypto_cb,
+ .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock),
+ .hash_lock = __RW_LOCK_UNLOCKED(lrng_drng_atomic.hash_lock)
+};
+
+/********************************** Helper ************************************/
+
+bool lrng_get_available(void)
+{
+ return likely(atomic_read(&lrng_avail));
+}
+
+void lrng_set_available(void)
+{
+ atomic_set(&lrng_avail, 1);
+}
+
+struct lrng_drng *lrng_drng_init_instance(void)
+{
+ return &lrng_drng_init;
+}
+
+struct lrng_drng *lrng_drng_atomic_instance(void)
+{
+ return &lrng_drng_atomic;
+}
+
+void lrng_drng_reset(struct lrng_drng *drng)
+{
+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH);
+ drng->last_seeded = jiffies;
+ drng->fully_seeded = false;
+ drng->force_reseed = true;
+ pr_debug("reset DRNG\n");
+}
+
+/* Initialize the default DRNG during boot */
+static void lrng_drng_seed(struct lrng_drng *drng);
+void lrng_drngs_init_cc20(bool force_seed)
+{
+ unsigned long flags = 0;
+
+ if (lrng_get_available())
+ return;
+
+ lrng_drng_lock(&lrng_drng_init, &flags);
+ if (lrng_get_available()) {
+ lrng_drng_unlock(&lrng_drng_init, &flags);
+ if (force_seed)
+ goto seed;
+ return;
+ }
+
+ lrng_drng_reset(&lrng_drng_init);
+ lrng_cc20_init_state(&chacha20);
+ lrng_drng_unlock(&lrng_drng_init, &flags);
+
+ lrng_drng_lock(&lrng_drng_atomic, &flags);
+ lrng_drng_reset(&lrng_drng_atomic);
+ /*
+ * We do not initialize the state of the atomic DRNG as it is identical
+ * to the DRNG at this point.
+ */
+ lrng_drng_unlock(&lrng_drng_atomic, &flags);
+
+ lrng_set_available();
+
+seed:
+ /* Seed the DRNG with any entropy available */
+ if (!lrng_pool_trylock()) {
+ lrng_drng_seed(&lrng_drng_init);
+ pr_info("ChaCha20 core initialized with first seeding\n");
+ lrng_pool_unlock();
+ } else {
+ pr_info("ChaCha20 core initialized without seeding\n");
+ }
+}
+
+bool lrng_sp80090c_compliant(void)
+{
+ if (!IS_ENABLED(CONFIG_LRNG_OVERSAMPLE_ENTROPY_SOURCES))
+ return false;
+
+ /* Entropy source hash must be capable of transporting enough entropy */
+ if (lrng_get_digestsize() <
+ (lrng_security_strength() + CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS))
+ return false;
+
+ /* SP800-90C only requested in FIPS mode */
+ return fips_enabled;
+}
+
+/************************* Random Number Generation ***************************/
+
+/* Inject a data buffer into the DRNG */
+static void lrng_drng_inject(struct lrng_drng *drng,
+ const u8 *inbuf, u32 inbuflen)
+{
+ const char *drng_type = unlikely(drng == &lrng_drng_atomic) ?
+ "atomic" : "regular";
+ unsigned long flags = 0;
+
+ BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX);
+ pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen);
+ lrng_drng_lock(drng, &flags);
+ if (drng->crypto_cb->lrng_drng_seed_helper(drng->drng,
+ inbuf, inbuflen) < 0) {
+ pr_warn("seeding of %s DRNG failed\n", drng_type);
+ atomic_set(&drng->requests, 1);
+ } else {
+ pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n",
+ drng_type,
+ (time_after(jiffies, drng->last_seeded) ?
+ (jiffies - drng->last_seeded) : 0) / HZ,
+ (LRNG_DRNG_RESEED_THRESH -
+ atomic_read(&drng->requests)));
+ drng->last_seeded = jiffies;
+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH);
+ drng->force_reseed = false;
+
+ if (drng->drng == lrng_drng_atomic.drng) {
+ lrng_drng_atomic.last_seeded = jiffies;
+ atomic_set(&lrng_drng_atomic.requests,
+ LRNG_DRNG_RESEED_THRESH);
+ lrng_drng_atomic.force_reseed = false;
+ }
+ }
+ lrng_drng_unlock(drng, &flags);
+}
+
+/*
+ * Perform the seeding of the DRNG with data from noise source
+ */
+static inline void _lrng_drng_seed(struct lrng_drng *drng)
+{
+ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN);
+
+ lrng_fill_seed_buffer(&seedbuf, lrng_get_seed_entropy_osr());
+ lrng_init_ops(&seedbuf);
+ lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf));
+
+ if (!drng->fully_seeded) {
+ drng->fully_seeded = lrng_fully_seeded(&seedbuf);
+ if (drng->fully_seeded)
+ pr_debug("DRNG fully seeded\n");
+ }
+ memzero_explicit(&seedbuf, sizeof(seedbuf));
+}
+
+static int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen);
+static void lrng_drng_seed(struct lrng_drng *drng)
+{
+ _lrng_drng_seed(drng);
+
+ BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS >
+ LRNG_DRNG_SECURITY_STRENGTH_BITS);
+
+ /*
+ * Reseed atomic DRNG from current DRNG,
+ *
+ * We can obtain random numbers from DRNG as the lock type
+ * chosen by lrng_drng_get is usable with the current caller.
+ */
+ if ((drng->drng != lrng_drng_atomic.drng) &&
+ (lrng_drng_atomic.force_reseed ||
+ atomic_read(&lrng_drng_atomic.requests) <= 0 ||
+ time_after(jiffies, lrng_drng_atomic.last_seeded +
+ lrng_drng_reseed_max_time * HZ))) {
+ u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES]
+ __aligned(LRNG_KCAPI_ALIGN);
+ int ret = lrng_drng_get(drng, seedbuf, sizeof(seedbuf));
+
+ if (ret < 0) {
+ pr_warn("Error generating random numbers for atomic DRNG: %d\n",
+ ret);
+ } else {
+ lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret);
+ }
+ memzero_explicit(&seedbuf, sizeof(seedbuf));
+ }
+}
+
+static inline void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node)
+{
+ pr_debug("reseed triggered by interrupt noise source for DRNG on NUMA node %d\n",
+ node);
+ lrng_drng_seed(drng);
+ if (drng->fully_seeded) {
+ /* Prevent reseed storm */
+ drng->last_seeded += node * 100 * HZ;
+ /* Prevent draining of pool on idle systems */
+ lrng_drng_reseed_max_time += 100;
+ }
+}
+
+/*
+ * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work()
+ */
+void lrng_drng_seed_work(struct work_struct *dummy)
+{
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ u32 node;
+
+ if (lrng_drng) {
+ for_each_online_node(node) {
+ struct lrng_drng *drng = lrng_drng[node];
+
+ if (drng && !drng->fully_seeded) {
+ _lrng_drng_seed_work(drng, node);
+ goto out;
+ }
+ }
+ } else {
+ if (!lrng_drng_init.fully_seeded) {
+ _lrng_drng_seed_work(&lrng_drng_init, 0);
+ goto out;
+ }
+ }
+
+ lrng_pool_all_numa_nodes_seeded(true);
+
+out:
+ /* Allow the seeding operation to be called again */
+ lrng_pool_unlock();
+}
+
+/* Force all DRNGs to reseed before next generation */
+void lrng_drng_force_reseed(void)
+{
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ u32 node;
+
+ if (!lrng_drng) {
+ lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded;
+ pr_debug("force reseed of initial DRNG\n");
+ return;
+ }
+ for_each_online_node(node) {
+ struct lrng_drng *drng = lrng_drng[node];
+
+ if (!drng)
+ continue;
+
+ drng->force_reseed = drng->fully_seeded;
+ pr_debug("force reseed of DRNG on node %u\n", node);
+ }
+ lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded;
+}
+
+/**
+ * lrng_drng_get() - Get random data out of the DRNG which is reseeded
+ * frequently.
+ *
+ * @outbuf: buffer for storing random data
+ * @outbuflen: length of outbuf
+ *
+ * Return:
+ * * < 0 in error case (DRNG generation or update failed)
+ * * >=0 returning the returned number of bytes
+ */
+static int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen)
+{
+ unsigned long flags = 0;
+ u32 processed = 0;
+
+ if (!outbuf || !outbuflen)
+ return 0;
+
+ outbuflen = min_t(size_t, outbuflen, INT_MAX);
+
+ lrng_drngs_init_cc20(false);
+
+ while (outbuflen) {
+ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE);
+ int ret;
+
+ /* All but the atomic DRNG are seeded during generation */
+ if (atomic_dec_and_test(&drng->requests) ||
+ drng->force_reseed ||
+ time_after(jiffies, drng->last_seeded +
+ lrng_drng_reseed_max_time * HZ)) {
+ if (likely(drng != &lrng_drng_atomic)) {
+ if (lrng_pool_trylock()) {
+ atomic_set(&drng->requests, 1);
+ } else {
+ lrng_drng_seed(drng);
+ lrng_pool_unlock();
+ }
+ }
+ }
+
+ lrng_drng_lock(drng, &flags);
+ ret = drng->crypto_cb->lrng_drng_generate_helper(
+ drng->drng, outbuf + processed, todo);
+ lrng_drng_unlock(drng, &flags);
+ if (ret <= 0) {
+ pr_warn("getting random data from DRNG failed (%d)\n",
+ ret);
+ return -EFAULT;
+ }
+ processed += ret;
+ outbuflen -= ret;
+ }
+
+ return processed;
+}
+
+int lrng_drng_get_atomic(u8 *outbuf, u32 outbuflen)
+{
+ return lrng_drng_get(&lrng_drng_atomic, outbuf, outbuflen);
+}
+
+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen)
+{
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ struct lrng_drng *drng = &lrng_drng_init;
+ int node = numa_node_id();
+
+ might_sleep();
+
+ if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded)
+ drng = lrng_drng[node];
+
+ return lrng_drng_get(drng, outbuf, outbuflen);
+}
+
+/* Reset LRNG such that all existing entropy is gone */
+static void _lrng_reset(struct work_struct *work)
+{
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ unsigned long flags = 0;
+
+ if (!lrng_drng) {
+ lrng_drng_lock(&lrng_drng_init, &flags);
+ lrng_drng_reset(&lrng_drng_init);
+ lrng_drng_unlock(&lrng_drng_init, &flags);
+ } else {
+ u32 node;
+
+ for_each_online_node(node) {
+ struct lrng_drng *drng = lrng_drng[node];
+
+ if (!drng)
+ continue;
+ lrng_drng_lock(drng, &flags);
+ lrng_drng_reset(drng);
+ lrng_drng_unlock(drng, &flags);
+ }
+ }
+ lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS);
+
+ lrng_reset_state();
+}
+
+static DECLARE_WORK(lrng_reset_work, _lrng_reset);
+
+void lrng_reset(void)
+{
+ schedule_work(&lrng_reset_work);
+}
+
+/***************************** Initialize LRNG *******************************/
+
+static int __init lrng_init(void)
+{
+ lrng_drngs_init_cc20(false);
+
+ lrng_drngs_numa_alloc();
+ return 0;
+}
+
+late_initcall(lrng_init);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
+MODULE_DESCRIPTION("Linux Random Number Generator");
new file mode 100644
@@ -0,0 +1,650 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG User and kernel space interfaces
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/freezer.h>
+#include <linux/fs.h>
+#include <linux/genhd.h>
+#include <linux/hw_random.h>
+#include <linux/kthread.h>
+#include <linux/poll.h>
+#include <linux/preempt.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/syscalls.h>
+#include <linux/timex.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/random.h>
+
+#include "lrng_internal.h"
+
+/*
+ * If the entropy count falls under this number of bits, then we
+ * should wake up processes which are selecting or polling on write
+ * access to /dev/random.
+ */
+u32 lrng_write_wakeup_bits = LRNG_WRITE_WAKEUP_ENTROPY;
+
+static LIST_HEAD(lrng_ready_list);
+static DEFINE_SPINLOCK(lrng_ready_list_lock);
+
+static DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait);
+static DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait);
+static struct fasync_struct *fasync;
+
+struct ctl_table random_table[];
+
+/********************************** Helper ***********************************/
+
+/* Is the DRNG seed level too low? */
+static inline bool lrng_need_entropy(void)
+{
+ return (lrng_avail_aux_entropy() < lrng_write_wakeup_bits);
+}
+
+void lrng_writer_wakeup(void)
+{
+ if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) {
+ wake_up_interruptible(&lrng_write_wait);
+ kill_fasync(&fasync, SIGIO, POLL_OUT);
+ }
+}
+
+void lrng_init_wakeup(void)
+{
+ wake_up_all(&lrng_init_wait);
+ kill_fasync(&fasync, SIGIO, POLL_IN);
+}
+
+/**
+ * lrng_process_ready_list() - Ping all kernel internal callers waiting until
+ * the DRNG is completely initialized to inform that the DRNG reached that
+ * seed level.
+ *
+ * When the SP800-90B testing is enabled, the ping only happens if the SP800-90B
+ * startup health tests are completed. This implies that kernel internal
+ * callers always have an SP800-90B compliant noise source when being
+ * pinged.
+ */
+void lrng_process_ready_list(void)
+{
+ unsigned long flags;
+ struct random_ready_callback *rdy, *tmp;
+
+ if (!lrng_state_operational())
+ return;
+
+ spin_lock_irqsave(&lrng_ready_list_lock, flags);
+ list_for_each_entry_safe(rdy, tmp, &lrng_ready_list, list) {
+ struct module *owner = rdy->owner;
+
+ list_del_init(&rdy->list);
+ rdy->func(rdy);
+ module_put(owner);
+ }
+ spin_unlock_irqrestore(&lrng_ready_list_lock, flags);
+}
+
+void lrng_debug_report_seedlevel(const char *name)
+{
+#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
+ static void *previous = NULL;
+ void *caller = (void *) _RET_IP_;
+
+ if (READ_ONCE(previous) == caller)
+ return;
+
+ if (!lrng_state_min_seeded())
+ pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n",
+ caller, name, lrng_avail_entropy());
+
+ WRITE_ONCE(previous, caller);
+#endif
+}
+
+/************************ LRNG kernel input interfaces ************************/
+
+/**
+ * add_hwgenerator_randomness() - Interface for in-kernel drivers of true
+ * hardware RNGs.
+ *
+ * Those devices may produce endless random bits and will be throttled
+ * when our pool is full.
+ *
+ * @buffer: buffer holding the entropic data from HW noise sources to be used to
+ * insert into entropy pool.
+ * @count: length of buffer
+ * @entropy_bits: amount of entropy in buffer (value is in bits)
+ */
+void add_hwgenerator_randomness(const char *buffer, size_t count,
+ size_t entropy_bits)
+{
+ /*
+ * Suspend writing if we are fully loaded with entropy.
+ * We'll be woken up again once below lrng_write_wakeup_thresh,
+ * or when the calling thread is about to terminate.
+ */
+ wait_event_interruptible(lrng_write_wait,
+ lrng_need_entropy() ||
+ lrng_state_exseed_allow(lrng_noise_source_hw) ||
+ kthread_should_stop());
+ lrng_state_exseed_set(lrng_noise_source_hw, false);
+ lrng_pool_insert_aux(buffer, count, entropy_bits);
+}
+EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
+
+/**
+ * add_bootloader_randomness() - Handle random seed passed by bootloader.
+ *
+ * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
+ * it would be regarded as device data.
+ * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
+ *
+ * @buf: buffer holding the entropic data from HW noise sources to be used to
+ * insert into entropy pool.
+ * @size: length of buffer
+ */
+void add_bootloader_randomness(const void *buf, unsigned int size)
+{
+ lrng_pool_insert_aux(buf, size,
+ IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER) ?
+ size * 8 : 0);
+}
+EXPORT_SYMBOL_GPL(add_bootloader_randomness);
+
+/*
+ * Callback for HID layer -- use the HID event values to stir the entropy pool
+ */
+void add_input_randomness(unsigned int type, unsigned int code,
+ unsigned int value)
+{
+ static unsigned char last_value;
+
+ /* ignore autorepeat and the like */
+ if (value == last_value)
+ return;
+
+ last_value = value;
+
+ lrng_pcpu_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value);
+}
+EXPORT_SYMBOL_GPL(add_input_randomness);
+
+/**
+ * add_device_randomness() - Add device- or boot-specific data to the entropy
+ * pool to help initialize it.
+ *
+ * None of this adds any entropy; it is meant to avoid the problem of
+ * the entropy pool having similar initial state across largely
+ * identical devices.
+ *
+ * @buf: buffer holding the entropic data from HW noise sources to be used to
+ * insert into entropy pool.
+ * @size: length of buffer
+ */
+void add_device_randomness(const void *buf, unsigned int size)
+{
+ lrng_pool_insert_aux((u8 *)buf, size, 0);
+}
+EXPORT_SYMBOL(add_device_randomness);
+
+#ifdef CONFIG_BLOCK
+void rand_initialize_disk(struct gendisk *disk) { }
+void add_disk_randomness(struct gendisk *disk) { }
+EXPORT_SYMBOL(add_disk_randomness);
+#endif
+
+/**
+ * del_random_ready_callback() - Delete a previously registered readiness
+ * callback function.
+ *
+ * @rdy: callback definition that was registered initially
+ */
+void del_random_ready_callback(struct random_ready_callback *rdy)
+{
+ unsigned long flags;
+ struct module *owner = NULL;
+
+ spin_lock_irqsave(&lrng_ready_list_lock, flags);
+ if (!list_empty(&rdy->list)) {
+ list_del_init(&rdy->list);
+ owner = rdy->owner;
+ }
+ spin_unlock_irqrestore(&lrng_ready_list_lock, flags);
+
+ module_put(owner);
+}
+EXPORT_SYMBOL(del_random_ready_callback);
+
+/**
+ * add_random_ready_callback() - Add a callback function that will be invoked
+ * when the DRNG is fully initialized and seeded.
+ *
+ * @rdy: callback definition to be invoked when the LRNG is seeded
+ *
+ * Return:
+ * * 0 if callback is successfully added
+ * * -EALREADY if pool is already initialised (callback not called)
+ * * -ENOENT if module for callback is not alive
+ */
+int add_random_ready_callback(struct random_ready_callback *rdy)
+{
+ struct module *owner;
+ unsigned long flags;
+ int err = -EALREADY;
+
+ if (likely(lrng_state_operational()))
+ return err;
+
+ owner = rdy->owner;
+ if (!try_module_get(owner))
+ return -ENOENT;
+
+ spin_lock_irqsave(&lrng_ready_list_lock, flags);
+ if (lrng_state_operational())
+ goto out;
+
+ owner = NULL;
+
+ list_add(&rdy->list, &lrng_ready_list);
+ err = 0;
+
+out:
+ spin_unlock_irqrestore(&lrng_ready_list_lock, flags);
+
+ module_put(owner);
+
+ return err;
+}
+EXPORT_SYMBOL(add_random_ready_callback);
+
+/*********************** LRNG kernel output interfaces ************************/
+
+/**
+ * get_random_bytes() - Provider of cryptographic strong random numbers for
+ * kernel-internal usage.
+ *
+ * This function is appropriate for all in-kernel use cases. However,
+ * it will always use the ChaCha20 DRNG.
+ *
+ * @buf: buffer to store the random bytes
+ * @nbytes: size of the buffer
+ */
+void get_random_bytes(void *buf, int nbytes)
+{
+ lrng_drng_get_atomic((u8 *)buf, (u32)nbytes);
+ lrng_debug_report_seedlevel("get_random_bytes");
+}
+EXPORT_SYMBOL(get_random_bytes);
+
+/**
+ * get_random_bytes_full() - Provider of cryptographic strong random numbers
+ * for kernel-internal usage.
+ *
+ * This function is appropriate only for non-atomic use cases as this
+ * function may sleep. Though, it provides access to the full functionality
+ * of LRNG including the switchable DRNG support, that may support other
+ * DRNGs such as the SP800-90A DRBG.
+ *
+ * @buf: buffer to store the random bytes
+ * @nbytes: size of the buffer
+ */
+void get_random_bytes_full(void *buf, int nbytes)
+{
+ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes);
+ lrng_debug_report_seedlevel("get_random_bytes_full");
+}
+EXPORT_SYMBOL(get_random_bytes_full);
+
+/**
+ * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus
+ * guaranteed to supply cryptographically secure random numbers.
+ *
+ * This applies to: the /dev/urandom device, the get_random_bytes function,
+ * and the get_random_{u32,u64,int,long} family of functions. Using any of
+ * these functions without first calling this function forfeits the guarantee
+ * of security.
+ *
+ * Return:
+ * * 0 if the LRNG has been seeded.
+ * * -ERESTARTSYS if the function was interrupted by a signal.
+ */
+int wait_for_random_bytes(void)
+{
+ if (likely(lrng_state_min_seeded()))
+ return 0;
+ return wait_event_interruptible(lrng_init_wait,
+ lrng_state_min_seeded());
+}
+EXPORT_SYMBOL(wait_for_random_bytes);
+
+/**
+ * get_random_bytes_arch() - This function will use the architecture-specific
+ * hardware random number generator if it is available.
+ *
+ * The arch-specific hw RNG will almost certainly be faster than what we can
+ * do in software, but it is impossible to verify that it is implemented
+ * securely (as opposed, to, say, the AES encryption of a sequence number using
+ * a key known by the NSA). So it's useful if we need the speed, but only if
+ * we're willing to trust the hardware manufacturer not to have put in a back
+ * door.
+ *
+ * @buf: buffer allocated by caller to store the random data in
+ * @nbytes: length of outbuf
+ *
+ * Return: number of bytes filled in.
+ */
+int __must_check get_random_bytes_arch(void *buf, int nbytes)
+{
+ u8 *p = buf;
+
+ while (nbytes) {
+ unsigned long v;
+ int chunk = min_t(int, nbytes, sizeof(unsigned long));
+
+ if (!arch_get_random_long(&v))
+ break;
+
+ memcpy(p, &v, chunk);
+ p += chunk;
+ nbytes -= chunk;
+ }
+
+ if (nbytes)
+ lrng_drng_get_atomic((u8 *)p, (u32)nbytes);
+
+ return nbytes;
+}
+EXPORT_SYMBOL(get_random_bytes_arch);
+
+/*
+ * Returns whether or not the LRNG has been seeded.
+ *
+ * Returns: true if the urandom pool has been seeded.
+ * false if the urandom pool has not been seeded.
+ */
+bool rng_is_initialized(void)
+{
+ return lrng_state_operational();
+}
+EXPORT_SYMBOL(rng_is_initialized);
+
+/************************ LRNG user output interfaces *************************/
+
+static ssize_t lrng_read_common(char __user *buf, size_t nbytes)
+{
+ ssize_t ret = 0;
+ u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN);
+ u8 *tmp_large = NULL, *tmp = tmpbuf;
+ u32 tmplen = sizeof(tmpbuf);
+
+ if (nbytes == 0)
+ return 0;
+
+ /*
+ * Satisfy large read requests -- as the common case are smaller
+ * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for
+ * those by using the stack variable of tmpbuf.
+ */
+ if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) {
+ tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE);
+ tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL);
+ if (!tmp_large)
+ tmplen = sizeof(tmpbuf);
+ else
+ tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN);
+ }
+
+ while (nbytes) {
+ u32 todo = min_t(u32, nbytes, tmplen);
+ int rc = 0;
+
+ /* Reschedule if we received a large request. */
+ if ((tmp_large) && need_resched()) {
+ if (signal_pending(current)) {
+ if (ret == 0)
+ ret = -ERESTARTSYS;
+ break;
+ }
+ schedule();
+ }
+
+ rc = lrng_drng_get_sleep(tmp, todo);
+ if (rc <= 0) {
+ if (rc < 0)
+ ret = rc;
+ break;
+ }
+ if (copy_to_user(buf, tmp, rc)) {
+ ret = -EFAULT;
+ break;
+ }
+
+ nbytes -= rc;
+ buf += rc;
+ ret += rc;
+ }
+
+ /* Wipe data just returned from memory */
+ if (tmp_large)
+ kfree_sensitive(tmp_large);
+ else
+ memzero_explicit(tmpbuf, sizeof(tmpbuf));
+
+ return ret;
+}
+
+static ssize_t
+lrng_read_common_block(int nonblock, char __user *buf, size_t nbytes)
+{
+ if (nbytes == 0)
+ return 0;
+
+ if (unlikely(!lrng_state_operational())) {
+ int ret;
+
+ if (nonblock)
+ return -EAGAIN;
+
+ ret = wait_event_interruptible(lrng_init_wait,
+ lrng_state_operational());
+ if (unlikely(ret))
+ return ret;
+ }
+
+ return lrng_read_common(buf, nbytes);
+}
+
+static ssize_t lrng_drng_read_block(struct file *file, char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ return lrng_read_common_block(file->f_flags & O_NONBLOCK, buf, nbytes);
+}
+
+static __poll_t lrng_random_poll(struct file *file, poll_table *wait)
+{
+ __poll_t mask;
+
+ poll_wait(file, &lrng_init_wait, wait);
+ poll_wait(file, &lrng_write_wait, wait);
+ mask = 0;
+ if (lrng_state_operational())
+ mask |= EPOLLIN | EPOLLRDNORM;
+ if (lrng_need_entropy() ||
+ lrng_state_exseed_allow(lrng_noise_source_user))
+ mask |= EPOLLOUT | EPOLLWRNORM;
+ return mask;
+}
+
+static ssize_t lrng_drng_write_common(const char __user *buffer, size_t count,
+ u32 entropy_bits)
+{
+ ssize_t ret = 0;
+ u8 buf[64] __aligned(LRNG_KCAPI_ALIGN);
+ const char __user *p = buffer;
+ u32 orig_entropy_bits = entropy_bits;
+
+ if (!lrng_get_available())
+ return -EAGAIN;
+
+ count = min_t(size_t, count, INT_MAX);
+ while (count > 0) {
+ size_t bytes = min_t(size_t, count, sizeof(buf));
+ u32 ent = min_t(u32, bytes<<3, entropy_bits);
+
+ if (copy_from_user(&buf, p, bytes))
+ return -EFAULT;
+ /* Inject data into entropy pool */
+ lrng_pool_insert_aux(buf, bytes, ent);
+
+ count -= bytes;
+ p += bytes;
+ ret += bytes;
+ entropy_bits -= ent;
+
+ cond_resched();
+ }
+
+ /* Force reseed of DRNG during next data request. */
+ if (!orig_entropy_bits)
+ lrng_drng_force_reseed();
+
+ return ret;
+}
+
+static ssize_t lrng_drng_read(struct file *file, char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ if (!lrng_state_min_seeded())
+ pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n",
+ current->comm, nbytes);
+ else if (!lrng_state_operational())
+ pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n",
+ current->comm, nbytes);
+
+ return lrng_read_common(buf, nbytes);
+}
+
+static ssize_t lrng_drng_write(struct file *file, const char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ return lrng_drng_write_common(buffer, count, 0);
+}
+
+static long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+{
+ u32 digestsize_bits;
+ int size, ent_count_bits;
+ int __user *p = (int __user *)arg;
+
+ switch (cmd) {
+ case RNDGETENTCNT:
+ ent_count_bits = lrng_avail_entropy();
+ if (put_user(ent_count_bits, p))
+ return -EFAULT;
+ return 0;
+ case RNDADDTOENTCNT:
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ if (get_user(ent_count_bits, p))
+ return -EFAULT;
+ ent_count_bits = (int)lrng_avail_aux_entropy() + ent_count_bits;
+ if (ent_count_bits < 0)
+ ent_count_bits = 0;
+ digestsize_bits = lrng_get_digestsize();
+ if (ent_count_bits > digestsize_bits)
+ ent_count_bits = digestsize_bits;
+ lrng_pool_set_entropy(ent_count_bits);
+ return 0;
+ case RNDADDENTROPY:
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ if (get_user(ent_count_bits, p++))
+ return -EFAULT;
+ if (ent_count_bits < 0)
+ return -EINVAL;
+ if (get_user(size, p++))
+ return -EFAULT;
+ if (size < 0)
+ return -EINVAL;
+ lrng_state_exseed_set(lrng_noise_source_user, false);
+ /* there cannot be more entropy than data */
+ ent_count_bits = min(ent_count_bits, size<<3);
+ return lrng_drng_write_common((const char __user *)p, size,
+ ent_count_bits);
+ case RNDZAPENTCNT:
+ case RNDCLEARPOOL:
+ /* Clear the entropy pool counter. */
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ lrng_pool_set_entropy(0);
+ return 0;
+ case RNDRESEEDCRNG:
+ /*
+ * We leave the capability check here since it is present
+ * in the upstream's RNG implementation. Yet, user space
+ * can trigger a reseed as easy as writing into /dev/random
+ * or /dev/urandom where no privilege is needed.
+ */
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ /* Force a reseed of all DRNGs */
+ lrng_drng_force_reseed();
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int lrng_fasync(int fd, struct file *filp, int on)
+{
+ return fasync_helper(fd, filp, on, &fasync);
+}
+
+const struct file_operations random_fops = {
+ .read = lrng_drng_read_block,
+ .write = lrng_drng_write,
+ .poll = lrng_random_poll,
+ .unlocked_ioctl = lrng_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .fasync = lrng_fasync,
+ .llseek = noop_llseek,
+};
+
+const struct file_operations urandom_fops = {
+ .read = lrng_drng_read,
+ .write = lrng_drng_write,
+ .unlocked_ioctl = lrng_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .fasync = lrng_fasync,
+ .llseek = noop_llseek,
+};
+
+SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
+ unsigned int, flags)
+{
+ if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE))
+ return -EINVAL;
+
+ /*
+ * Requesting insecure and blocking randomness at the same time makes
+ * no sense.
+ */
+ if ((flags &
+ (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM))
+ return -EINVAL;
+
+ if (count > INT_MAX)
+ count = INT_MAX;
+
+ if (flags & GRND_INSECURE)
+ return lrng_drng_read(NULL, buf, count, NULL);
+
+ return lrng_read_common_block(flags & GRND_NONBLOCK, buf, count);
+}
new file mode 100644
@@ -0,0 +1,416 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (C) 2018 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#ifndef _LRNG_INTERNAL_H
+#define _LRNG_INTERNAL_H
+
+#include <crypto/sha1.h>
+#include <crypto/sha2.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+/*************************** General LRNG parameter ***************************/
+
+/* Security strength of LRNG -- this must match DRNG security strength */
+#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32
+#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8)
+#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */
+
+/*
+ * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is
+ * considered a safer margin.
+ *
+ * This value is allowed to be changed.
+ */
+#define LRNG_DRNG_MAX_REQSIZE (1<<12)
+
+/*
+ * SP800-90A defines a maximum number of requests between reseeds of 2^48.
+ * The given value is considered a much safer margin, balancing requests for
+ * frequent reseeds with the need to conserve entropy. This value MUST NOT be
+ * larger than INT_MAX because it is used in an atomic_t.
+ *
+ * This value is allowed to be changed.
+ */
+#define LRNG_DRNG_RESEED_THRESH (1<<20)
+
+/*
+ * Number of interrupts to be recorded to assume that DRNG security strength
+ * bits of entropy are received.
+ * Note: a value below the DRNG security strength should not be defined as this
+ * may imply the DRNG can never be fully seeded in case other noise
+ * sources are unavailable.
+ *
+ * This value is allowed to be changed.
+ */
+#define LRNG_IRQ_ENTROPY_BITS CONFIG_LRNG_IRQ_ENTROPY_RATE
+
+/*
+ * Min required seed entropy is 128 bits covering the minimum entropy
+ * requirement of SP800-131A and the German BSI's TR02102.
+ *
+ * This value is allowed to be changed.
+ */
+#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS
+#define LRNG_MIN_SEED_ENTROPY_BITS 128
+#define LRNG_INIT_ENTROPY_BITS 32
+
+/*
+ * Wakeup value
+ *
+ * This value is allowed to be changed but must not be larger than the
+ * digest size of the hash operation used update the aux_pool.
+ */
+#ifdef CONFIG_CRYPTO_LIB_SHA256
+# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE
+#else
+# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE
+#endif
+#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE
+
+/*
+ * If the switching support is configured, we must provide support up to
+ * the largest digest size. Without switching support, we know it is only
+ * the built-in digest size.
+ */
+#ifdef CONFIG_LRNG_DRNG_SWITCH
+# define LRNG_MAX_DIGESTSIZE 64
+#else
+# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE
+#endif
+
+/*
+ * Oversampling factor of IRQ events to obtain
+ * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a
+ * high-resolution time stamp is not available. In this case, jiffies and
+ * register contents are used to fill the entropy pool. These noise sources
+ * are much less entropic than the high-resolution timer. The entropy content
+ * is the entropy content assumed with LRNG_IRQ_ENTROPY_BITS divided by
+ * LRNG_IRQ_OVERSAMPLING_FACTOR.
+ *
+ * This value is allowed to be changed.
+ */
+#define LRNG_IRQ_OVERSAMPLING_FACTOR 10
+
+/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */
+#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN
+
+/*
+ * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK
+ * as it will be casted into a struct shash_desc.
+ */
+#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE)
+
+/************************ Default DRNG implementation *************************/
+
+extern struct chacha20_state chacha20;
+extern const struct lrng_crypto_cb lrng_cc20_crypto_cb;
+void lrng_cc20_init_state(struct chacha20_state *state);
+
+/********************************** /proc *************************************/
+
+static inline void lrng_pool_inc_numa_node(void) { }
+
+/****************************** LRNG interfaces *******************************/
+
+extern u32 lrng_write_wakeup_bits;
+extern int lrng_drng_reseed_max_time;
+
+void lrng_writer_wakeup(void);
+void lrng_init_wakeup(void);
+void lrng_debug_report_seedlevel(const char *name);
+void lrng_process_ready_list(void);
+
+/* External interface to use of the switchable DRBG inside the kernel */
+void get_random_bytes_full(void *buf, int nbytes);
+
+/************************* Jitter RNG Entropy Source **************************/
+
+#ifdef CONFIG_LRNG_JENT
+u32 lrng_get_jent(u8 *outbuf, u32 requested_bits);
+u32 lrng_jent_entropylevel(u32 requested_bits);
+#else /* CONFIG_CRYPTO_JITTERENTROPY */
+static inline u32 lrng_get_jent(u8 *outbuf, u32 requested_bits) { return 0; }
+static inline u32 lrng_jent_entropylevel(u32 requested_bits) { return 0; }
+#endif /* CONFIG_CRYPTO_JITTERENTROPY */
+
+/************************** CPU-based Entropy Source **************************/
+
+static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits)
+{
+ /* Obtain entropy statement */
+ ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS;
+ /* Cap entropy to buffer size in bits */
+ ent_bits = min_t(u32, ent_bits, requested_bits);
+ return ent_bits;
+}
+
+u32 lrng_get_arch(u8 *outbuf, u32 requested_bits);
+u32 lrng_archrandom_entropylevel(u32 requested_bits);
+
+/************************** Interrupt Entropy Source **************************/
+
+bool lrng_pcpu_continuous_compression_state(void);
+void lrng_pcpu_reset(void);
+u32 lrng_pcpu_avail_pool_size(void);
+u32 lrng_pcpu_avail_entropy(void);
+int lrng_pcpu_switch_hash(int node,
+ const struct lrng_crypto_cb *new_cb, void *new_hash,
+ const struct lrng_crypto_cb *old_cb);
+u32 lrng_pcpu_pool_hash(u8 *outbuf, u32 requested_bits, bool fully_seeded);
+void lrng_pcpu_array_add_u32(u32 data);
+
+/****************************** DRNG processing *******************************/
+
+/* DRNG state handle */
+struct lrng_drng {
+ void *drng; /* DRNG handle */
+ void *hash; /* Hash handle */
+ const struct lrng_crypto_cb *crypto_cb; /* Crypto callbacks */
+ atomic_t requests; /* Number of DRNG requests */
+ unsigned long last_seeded; /* Last time it was seeded */
+ bool fully_seeded; /* Is DRNG fully seeded? */
+ bool force_reseed; /* Force a reseed */
+
+ /* Lock write operations on DRNG state, DRNG replacement of crypto_cb */
+ struct mutex lock;
+ spinlock_t spin_lock;
+ /* Lock *hash replacement - always take before DRNG lock */
+ rwlock_t hash_lock;
+};
+
+extern struct mutex lrng_crypto_cb_update;
+
+struct lrng_drng *lrng_drng_init_instance(void);
+struct lrng_drng *lrng_drng_atomic_instance(void);
+
+static __always_inline bool lrng_drng_is_atomic(struct lrng_drng *drng)
+{
+ return (drng->drng == lrng_drng_atomic_instance()->drng);
+}
+
+/* Lock the DRNG */
+static __always_inline void lrng_drng_lock(struct lrng_drng *drng,
+ unsigned long *flags)
+ __acquires(&drng->spin_lock)
+{
+ /* Use spin lock in case the atomic DRNG context is used */
+ if (lrng_drng_is_atomic(drng)) {
+ spin_lock_irqsave(&drng->spin_lock, *flags);
+
+ /*
+ * In case a lock transition happened while we were spinning,
+ * catch this case and use the new lock type.
+ */
+ if (!lrng_drng_is_atomic(drng)) {
+ spin_unlock_irqrestore(&drng->spin_lock, *flags);
+ __acquire(&drng->spin_lock);
+ mutex_lock(&drng->lock);
+ }
+ } else {
+ __acquire(&drng->spin_lock);
+ mutex_lock(&drng->lock);
+ }
+}
+
+/* Unlock the DRNG */
+static __always_inline void lrng_drng_unlock(struct lrng_drng *drng,
+ unsigned long *flags)
+ __releases(&drng->spin_lock)
+{
+ if (lrng_drng_is_atomic(drng)) {
+ spin_unlock_irqrestore(&drng->spin_lock, *flags);
+ } else {
+ mutex_unlock(&drng->lock);
+ __release(&drng->spin_lock);
+ }
+}
+
+void lrng_reset(void);
+void lrng_drngs_init_cc20(bool force_seed);
+bool lrng_sp80090c_compliant(void);
+
+static inline u32 lrng_compress_osr(void)
+{
+ return lrng_sp80090c_compliant() ? CONFIG_LRNG_OVERSAMPLE_ES_BITS : 0;
+}
+
+static inline u32 lrng_reduce_by_osr(u32 entropy_bits)
+{
+ u32 osr_bits = lrng_compress_osr();
+ return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0;
+}
+
+bool lrng_get_available(void);
+void lrng_set_available(void);
+void lrng_drng_reset(struct lrng_drng *drng);
+int lrng_drng_get_atomic(u8 *outbuf, u32 outbuflen);
+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen);
+void lrng_drng_force_reseed(void);
+void lrng_drng_seed_work(struct work_struct *dummy);
+
+static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; }
+static inline void lrng_drngs_numa_alloc(void) { return; }
+
+/************************* Entropy sources management *************************/
+
+enum lrng_external_noise_source {
+ lrng_noise_source_hw,
+ lrng_noise_source_user
+};
+
+u32 lrng_avail_aux_entropy(void);
+u32 lrng_get_digestsize(void);
+
+/* Obtain the security strength of the LRNG in bits */
+static inline u32 lrng_security_strength(void)
+{
+ /*
+ * We use a hash to read the entropy in the entropy pool. According to
+ * SP800-90B table 1, the entropy can be at most the digest size.
+ * Considering this together with the last sentence in section 3.1.5.1.2
+ * the security strength of a (approved) hash is equal to its output
+ * size. On the other hand the entropy cannot be larger than the
+ * security strength of the used DRBG.
+ */
+ return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize());
+}
+
+static inline u32 lrng_get_seed_entropy_osr(void)
+{
+ u32 requested_bits = lrng_security_strength();
+
+ /* Apply oversampling during initialization according to SP800-90C */
+ if (lrng_sp80090c_compliant())
+ requested_bits += CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS;
+ return requested_bits;
+}
+
+void lrng_set_entropy_thresh(u32 new);
+u32 lrng_avail_entropy(void);
+void lrng_reset_state(void);
+
+bool lrng_state_exseed_allow(enum lrng_external_noise_source source);
+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type);
+bool lrng_state_min_seeded(void);
+bool lrng_state_fully_seeded(void);
+bool lrng_state_operational(void);
+
+int lrng_pool_trylock(void);
+void lrng_pool_unlock(void);
+void lrng_pool_all_numa_nodes_seeded(bool set);
+bool lrng_pool_highres_timer(void);
+void lrng_pool_set_entropy(u32 entropy_bits);
+int lrng_aux_switch_hash(const struct lrng_crypto_cb *new_cb, void *new_hash,
+ const struct lrng_crypto_cb *old_cb);
+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits);
+void lrng_pool_add_entropy(void);
+
+struct entropy_buf {
+ u8 a[LRNG_DRNG_SECURITY_STRENGTH_BYTES +
+ (CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS >> 3)];
+ u8 b[LRNG_DRNG_SECURITY_STRENGTH_BYTES +
+ (CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS >> 3)];
+ u8 c[LRNG_DRNG_SECURITY_STRENGTH_BYTES +
+ (CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS >> 3)];
+ u8 d[LRNG_DRNG_SECURITY_STRENGTH_BYTES +
+ (CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS >> 3)];
+ u32 now, a_bits, b_bits, c_bits, d_bits;
+};
+
+bool lrng_fully_seeded(struct entropy_buf *eb);
+void lrng_unset_operational(void);
+void lrng_fill_seed_buffer(struct entropy_buf *entropy_buf, u32 requested_bits);
+void lrng_init_ops(struct entropy_buf *eb);
+
+/************************** Health Test linking code **************************/
+
+enum lrng_health_res {
+ lrng_health_pass, /* Health test passes on time stamp */
+ lrng_health_fail_use, /* Time stamp unhealthy, but mix in */
+ lrng_health_fail_drop /* Time stamp unhealthy, drop it */
+};
+
+#ifdef CONFIG_LRNG_HEALTH_TESTS
+bool lrng_sp80090b_startup_complete(void);
+bool lrng_sp80090b_compliant(void);
+
+enum lrng_health_res lrng_health_test(u32 now_time);
+void lrng_health_disable(void);
+
+#else /* CONFIG_LRNG_HEALTH_TESTS */
+static inline bool lrng_sp80090b_startup_complete(void) { return true; }
+static inline bool lrng_sp80090b_compliant(void) { return false; }
+
+static inline enum lrng_health_res
+lrng_health_test(u32 now_time) { return lrng_health_pass; }
+static inline void lrng_health_disable(void) { }
+#endif /* CONFIG_LRNG_HEALTH_TESTS */
+
+/****************************** Helper code ***********************************/
+
+static inline u32 atomic_read_u32(atomic_t *v)
+{
+ return (u32)atomic_read(v);
+}
+
+/*************************** Auxiliary functions ******************************/
+
+void invalidate_batched_entropy(void);
+
+/***************************** Testing code ***********************************/
+
+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY
+bool lrng_raw_hires_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */
+static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY
+bool lrng_raw_jiffies_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */
+static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY
+bool lrng_raw_irq_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */
+static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY
+bool lrng_raw_irqflags_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY */
+static inline bool lrng_raw_irqflags_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_IRQFLAGS_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY
+bool lrng_raw_retip_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */
+static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY
+bool lrng_raw_regs_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */
+static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */
+
+#ifdef CONFIG_LRNG_RAW_ARRAY
+bool lrng_raw_array_entropy_store(u32 value);
+#else /* CONFIG_LRNG_RAW_ARRAY */
+static inline bool lrng_raw_array_entropy_store(u32 value) { return false; }
+#endif /* CONFIG_LRNG_RAW_ARRAY */
+
+#ifdef CONFIG_LRNG_IRQ_PERF
+bool lrng_perf_time(u32 start);
+#else /* CONFIG_LRNG_IRQ_PERF */
+static inline bool lrng_perf_time(u32 start) { return false; }
+#endif /*CONFIG_LRNG_IRQ_PERF */
+
+#endif /* _LRNG_INTERNAL_H */
new file mode 100644
@@ -0,0 +1,622 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG Entropy sources management
+ * LRNG Slow Entropy Source: Auxiliary entropy pool
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <asm/irq_regs.h>
+#include <linux/lrng.h>
+#include <linux/percpu.h>
+#include <linux/random.h>
+#include <linux/utsname.h>
+#include <linux/workqueue.h>
+
+#include "lrng_internal.h"
+#include "lrng_sw_noise.h"
+
+struct lrng_state {
+ bool can_invalidate; /* Can invalidate batched entropy? */
+ bool perform_seedwork; /* Can seed work be performed? */
+ bool lrng_operational; /* Is DRNG operational? */
+ bool lrng_fully_seeded; /* Is DRNG fully seeded? */
+ bool lrng_min_seeded; /* Is DRNG minimally seeded? */
+ bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */
+
+ /*
+ * To ensure that external entropy providers cannot dominate the
+ * internal noise sources but yet cannot be dominated by internal
+ * noise sources, the following booleans are intended to allow
+ * external to provide seed once when a DRNG reseed occurs. This
+ * triggering of external noise source is performed even when the
+ * entropy pool has sufficient entropy.
+ */
+ bool lrng_seed_hw; /* Allow HW to provide seed */
+ bool lrng_seed_user; /* Allow user space to provide seed */
+
+ atomic_t boot_entropy_thresh; /* Reseed threshold */
+ atomic_t reseed_in_progress; /* Flag for on executing reseed */
+ struct work_struct lrng_seed_work; /* (re)seed work queue */
+};
+
+/*
+ * This is the auxiliary pool
+ *
+ * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API
+ * cipher implementations of the hash functions used to read the pool: for some
+ * accelerated implementations, we need an alignment to avoid a realignment
+ * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto
+ * implementations.
+ */
+struct lrng_pool {
+ u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */
+ atomic_t aux_entropy_bits;
+ atomic_t digestsize; /* Digest size of used hash */
+ bool initialized; /* Aux pool initialized? */
+
+ /* Serialize read of entropy pool and update of aux pool */
+ spinlock_t lock;
+};
+
+static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = {
+ .aux_entropy_bits = ATOMIC_INIT(0),
+ .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE),
+ .initialized = false,
+ .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock)
+};
+
+static struct lrng_state lrng_state = {
+ false, false, false, false, false, false, true, true,
+ .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS),
+ .reseed_in_progress = ATOMIC_INIT(0),
+};
+
+/********************************** Helper ***********************************/
+
+/* External entropy provider is allowed to provide seed data */
+bool lrng_state_exseed_allow(enum lrng_external_noise_source source)
+{
+ if (source == lrng_noise_source_hw)
+ return lrng_state.lrng_seed_hw;
+ return lrng_state.lrng_seed_user;
+}
+
+/* Enable / disable external entropy provider to furnish seed */
+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type)
+{
+ if (source == lrng_noise_source_hw)
+ lrng_state.lrng_seed_hw = type;
+ else
+ lrng_state.lrng_seed_user = type;
+}
+
+static inline void lrng_state_exseed_allow_all(void)
+{
+ lrng_state_exseed_set(lrng_noise_source_hw, true);
+ lrng_state_exseed_set(lrng_noise_source_user, true);
+}
+
+/* Entropy in bits present in aux pool */
+u32 lrng_avail_aux_entropy(void)
+{
+ /* Cap available entropy with max entropy */
+ u32 avail_bits = min_t(u32, lrng_get_digestsize(),
+ atomic_read_u32(&lrng_pool.aux_entropy_bits));
+
+ /* Consider oversampling rate due to aux pool conditioning */
+ return lrng_reduce_by_osr(avail_bits);
+}
+
+/* Set the digest size of the used hash in bytes */
+static inline void lrng_set_digestsize(u32 digestsize)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0),
+ old_digestsize = lrng_get_digestsize();
+
+ atomic_set(&lrng_pool.digestsize, digestsize);
+
+ /*
+ * In case the new digest is larger than the old one, cap the available
+ * entropy to the old message digest used to process the existing data.
+ */
+ ent_bits = min_t(u32, ent_bits, old_digestsize);
+ atomic_add(ent_bits, &pool->aux_entropy_bits);
+}
+
+/* Obtain the digest size provided by the used hash in bits */
+u32 lrng_get_digestsize(void)
+{
+ return atomic_read_u32(&lrng_pool.digestsize) << 3;
+}
+
+/*
+ * Reading of the LRNG pool is only allowed by one caller. The reading is
+ * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken,
+ * the reseeding operation is in progress. The caller is not intended to wait
+ * but continue with its other operation.
+ */
+int lrng_pool_trylock(void)
+{
+ return atomic_cmpxchg(&lrng_state.reseed_in_progress, 0, 1);
+}
+
+void lrng_pool_unlock(void)
+{
+ atomic_set(&lrng_state.reseed_in_progress, 0);
+}
+
+/* Set new entropy threshold for reseeding during boot */
+void lrng_set_entropy_thresh(u32 new_entropy_bits)
+{
+ atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits);
+}
+
+/*
+ * Reset LRNG state - the entropy counters are reset, but the data that may
+ * or may not have entropy remains in the pools as this data will not hurt.
+ */
+void lrng_reset_state(void)
+{
+ atomic_set(&lrng_pool.aux_entropy_bits, 0);
+ lrng_pcpu_reset();
+ lrng_state.lrng_operational = false;
+ lrng_state.lrng_fully_seeded = false;
+ lrng_state.lrng_min_seeded = false;
+ lrng_state.all_online_numa_node_seeded = false;
+ pr_debug("reset LRNG\n");
+}
+
+/* Set flag that all DRNGs are fully seeded */
+void lrng_pool_all_numa_nodes_seeded(bool set)
+{
+ lrng_state.all_online_numa_node_seeded = set;
+}
+
+/* Return boolean whether LRNG reached minimally seed level */
+bool lrng_state_min_seeded(void)
+{
+ return lrng_state.lrng_min_seeded;
+}
+
+/* Return boolean whether LRNG reached fully seed level */
+bool lrng_state_fully_seeded(void)
+{
+ return lrng_state.lrng_fully_seeded;
+}
+
+/* Return boolean whether LRNG is considered fully operational */
+bool lrng_state_operational(void)
+{
+ return lrng_state.lrng_operational;
+}
+
+/* Policy to check whether entropy buffer contains full seeded entropy */
+bool lrng_fully_seeded(struct entropy_buf *eb)
+{
+ return ((eb->a_bits + eb->b_bits + eb->c_bits + eb->d_bits) >=
+ lrng_get_seed_entropy_osr());
+}
+
+/* Disable the fully seeded and operational mode */
+void lrng_unset_operational(void)
+{
+ lrng_pool_all_numa_nodes_seeded(false);
+ lrng_state.lrng_operational = false;
+ lrng_state.lrng_fully_seeded = false;
+}
+
+/* Policy to enable LRNG operational mode */
+static inline void lrng_set_operational(u32 external_es)
+{
+ if (lrng_state.lrng_fully_seeded &&
+ (lrng_sp80090b_startup_complete() ||
+ (lrng_get_seed_entropy_osr() <= external_es))) {
+ lrng_state.lrng_operational = true;
+ lrng_process_ready_list();
+ lrng_init_wakeup();
+ pr_info("LRNG fully operational\n");
+ }
+}
+
+/* Set entropy content in user-space controllable aux pool */
+void lrng_pool_set_entropy(u32 entropy_bits)
+{
+ atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits);
+}
+
+/* Available entropy in the entire LRNG considering all entropy sources */
+u32 lrng_avail_entropy(void)
+{
+ u32 ent_thresh = lrng_security_strength();
+
+ /*
+ * Apply oversampling during initialization according to SP800-90C as
+ * we request a larger buffer from the ES.
+ */
+ if (lrng_sp80090c_compliant() &&
+ !lrng_state.all_online_numa_node_seeded)
+ ent_thresh += CONFIG_LRNG_SEED_BUFFER_INIT_ADD_BITS;
+
+ return lrng_pcpu_avail_entropy() + lrng_avail_aux_entropy() +
+ lrng_archrandom_entropylevel(ent_thresh) +
+ lrng_jent_entropylevel(ent_thresh);
+}
+
+/**
+ * lrng_init_ops() - Set seed stages of LRNG
+ *
+ * Set the slow noise source reseed trigger threshold. The initial threshold
+ * is set to the minimum data size that can be read from the pool: a word. Upon
+ * reaching this value, the next seed threshold of 128 bits is set followed
+ * by 256 bits.
+ *
+ * @eb: buffer containing the size of entropy currently injected into DRNG
+ */
+void lrng_init_ops(struct entropy_buf *eb)
+{
+ struct lrng_state *state = &lrng_state;
+ u32 requested_bits, seed_bits, external_es;
+
+ if (state->lrng_operational)
+ return;
+
+ requested_bits = lrng_get_seed_entropy_osr();
+
+ /*
+ * Entropy provided by external entropy sources - if they provide
+ * the requested amount of entropy, unblock the interface.
+ */
+ external_es = eb->a_bits + eb->c_bits + eb->d_bits;
+ seed_bits = external_es + eb->b_bits;
+
+ /* DRNG is seeded with full security strength */
+ if (state->lrng_fully_seeded) {
+ lrng_set_operational(external_es);
+ lrng_set_entropy_thresh(requested_bits);
+ } else if (lrng_fully_seeded(eb)) {
+ if (state->can_invalidate)
+ invalidate_batched_entropy();
+
+ state->lrng_fully_seeded = true;
+ lrng_set_operational(external_es);
+ state->lrng_min_seeded = true;
+ pr_info("LRNG fully seeded with %u bits of entropy\n",
+ seed_bits);
+ lrng_set_entropy_thresh(requested_bits);
+ } else if (!state->lrng_min_seeded) {
+
+ /* DRNG is seeded with at least 128 bits of entropy */
+ if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) {
+ if (state->can_invalidate)
+ invalidate_batched_entropy();
+
+ state->lrng_min_seeded = true;
+ pr_info("LRNG minimally seeded with %u bits of entropy\n",
+ seed_bits);
+ lrng_set_entropy_thresh(requested_bits);
+ lrng_init_wakeup();
+
+ /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */
+ } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) {
+ pr_info("LRNG initial entropy level %u bits of entropy\n",
+ seed_bits);
+ lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS);
+ }
+ }
+}
+
+int __init rand_initialize(void)
+{
+ struct seed {
+ ktime_t time;
+ unsigned long data[(LRNG_MAX_DIGESTSIZE /
+ sizeof(unsigned long))];
+ struct new_utsname utsname;
+ } seed __aligned(LRNG_KCAPI_ALIGN);
+ unsigned int i;
+
+ BUILD_BUG_ON(LRNG_MAX_DIGESTSIZE % sizeof(unsigned long));
+
+ seed.time = ktime_get_real();
+
+ for (i = 0; i < ARRAY_SIZE(seed.data); i++) {
+ if (!arch_get_random_seed_long_early(&(seed.data[i])) &&
+ !arch_get_random_long_early(&seed.data[i]))
+ seed.data[i] = random_get_entropy();
+ }
+ memcpy(&seed.utsname, utsname(), sizeof(*(utsname())));
+
+ lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0);
+ memzero_explicit(&seed, sizeof(seed));
+
+ /* Initialize the seed work queue */
+ INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work);
+ lrng_state.perform_seedwork = true;
+
+ lrng_drngs_init_cc20(true);
+ invalidate_batched_entropy();
+
+ lrng_state.can_invalidate = true;
+
+ return 0;
+}
+
+/*
+ * Replace old with new hash for auxiliary pool handling
+ *
+ * Assumption: the caller must guarantee that the new_cb is available during the
+ * entire operation (e.g. it must hold the write lock against pointer updating).
+ */
+int lrng_aux_switch_hash(const struct lrng_crypto_cb *new_cb, void *new_hash,
+ const struct lrng_crypto_cb *old_cb)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool;
+ u8 digest[LRNG_MAX_DIGESTSIZE];
+ int ret;
+
+ if (!IS_ENABLED(CONFIG_LRNG_DRNG_SWITCH))
+ return -EOPNOTSUPP;
+
+ if (unlikely(!pool->initialized))
+ return 0;
+
+ /* Get the aux pool hash with old digest ... */
+ ret = old_cb->lrng_hash_final(shash, digest) ?:
+ /* ... re-initialize the hash with the new digest ... */
+ new_cb->lrng_hash_init(shash, new_hash) ?:
+ /*
+ * ... feed the old hash into the new state. We may feed
+ * uninitialized memory into the new state, but this is
+ * considered no issue and even good as we have some more
+ * uncertainty here.
+ */
+ new_cb->lrng_hash_update(shash, digest, sizeof(digest));
+ if (!ret) {
+ lrng_set_digestsize(new_cb->lrng_hash_digestsize(new_hash));
+ pr_debug("Re-initialize aux entropy pool with hash %s\n",
+ new_cb->lrng_hash_name());
+ }
+
+ memzero_explicit(digest, sizeof(digest));
+ return ret;
+}
+
+/*
+ * Insert data into auxiliary pool by hashing the input data together with
+ * the auxiliary pool. The message digest is the new state of the auxiliary
+ * pool.
+ */
+static int
+lrng_pool_insert_aux_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool;
+ struct lrng_drng *drng = lrng_drng_init_instance();
+ const struct lrng_crypto_cb *crypto_cb;
+ unsigned long flags;
+ void *hash;
+ int ret;
+
+ entropy_bits = min_t(u32, entropy_bits, inbuflen << 3);
+
+ read_lock_irqsave(&drng->hash_lock, flags);
+
+ crypto_cb = drng->crypto_cb;
+ hash = drng->hash;
+
+ if (unlikely(!pool->initialized)) {
+ ret = crypto_cb->lrng_hash_init(shash, hash);
+ if (ret)
+ goto out;
+ pool->initialized = true;
+ }
+
+ ret = crypto_cb->lrng_hash_update(shash, inbuf, inbuflen);
+ if (ret)
+ goto out;
+
+ /*
+ * Cap the available entropy to the hash output size compliant to
+ * SP800-90B section 3.1.5.1 table 1.
+ */
+ entropy_bits += atomic_read_u32(&pool->aux_entropy_bits);
+ atomic_set(&pool->aux_entropy_bits,
+ min_t(u32, entropy_bits,
+ crypto_cb->lrng_hash_digestsize(hash) << 3));
+
+out:
+ read_unlock_irqrestore(&drng->hash_lock, flags);
+ return ret;
+}
+
+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&pool->lock, flags);
+ ret = lrng_pool_insert_aux_locked(inbuf, inbuflen, entropy_bits);
+ spin_unlock_irqrestore(&pool->lock, flags);
+
+ lrng_pool_add_entropy();
+
+ return ret;
+}
+
+/* Hot code path during boot - mix data into entropy pool during boot */
+void lrng_pool_add_entropy(void)
+{
+ /*
+ * Once all DRNGs are fully seeded, the interrupt noise
+ * sources will not trigger any reseeding any more.
+ */
+ if (likely(lrng_state.all_online_numa_node_seeded))
+ return;
+
+ /* Only try to reseed if the DRNG is alive. */
+ if (!lrng_get_available())
+ return;
+
+ /* Only trigger the DRNG reseed if we have collected entropy. */
+ if (lrng_avail_entropy() <
+ atomic_read_u32(&lrng_state.boot_entropy_thresh))
+ return;
+
+ /* Ensure that the seeding only occurs once at any given time. */
+ if (lrng_pool_trylock())
+ return;
+
+ /* Seed the DRNG with IRQ noise. */
+ if (lrng_state.perform_seedwork)
+ schedule_work(&lrng_state.lrng_seed_work);
+ else
+ lrng_drng_seed_work(NULL);
+}
+
+/************************* Get data from entropy pool *************************/
+
+/**
+ * Get auxiliary entropy pool and its entropy content for seed buffer.
+ * Caller must hold lrng_pool.pool->lock.
+ * @outbuf: buffer to store data in with size requested_bits
+ * @requested_bits: Requested amount of entropy
+ * @return: amount of entropy in outbuf in bits.
+ */
+static inline u32 lrng_get_aux_pool(u8 *outbuf, u32 requested_bits)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool;
+ struct lrng_drng *drng = lrng_drng_init_instance();
+ const struct lrng_crypto_cb *crypto_cb;
+ unsigned long flags;
+ void *hash;
+ u32 collected_ent_bits, returned_ent_bits, unused_bits = 0,
+ digestsize;
+ u8 aux_output[LRNG_MAX_DIGESTSIZE];
+
+ if (unlikely(!pool->initialized))
+ return 0;
+
+ read_lock_irqsave(&drng->hash_lock, flags);
+
+ crypto_cb = drng->crypto_cb;
+ hash = drng->hash;
+ digestsize = crypto_cb->lrng_hash_digestsize(hash);
+
+ /* Ensure that no more than the size of aux_pool can be requested */
+ requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3));
+
+ /* Cap entropy with entropy counter from aux pool and the used digest */
+ collected_ent_bits = min_t(u32, digestsize << 3,
+ atomic_xchg_relaxed(&pool->aux_entropy_bits, 0));
+
+ /* We collected too much entropy and put the overflow back */
+ if (collected_ent_bits > (requested_bits + lrng_compress_osr())) {
+ /* Amount of bits we collected too much */
+ unused_bits = collected_ent_bits - requested_bits;
+ /* Put entropy back */
+ atomic_add(unused_bits, &pool->aux_entropy_bits);
+ /* Fix collected entropy */
+ collected_ent_bits = requested_bits;
+ }
+
+ /* Apply oversampling: discount requested oversampling rate */
+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits);
+
+ pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n",
+ returned_ent_bits, collected_ent_bits, unused_bits);
+
+ /* Get the digest for the aux pool to be returned to the caller ... */
+ if (crypto_cb->lrng_hash_final(shash, aux_output) ||
+ /*
+ * ... and re-initialize the aux state. Do not add the aux pool
+ * digest for backward secrecy as it will be added with the
+ * insertion of the complete seed buffer after it has been filled.
+ */
+ crypto_cb->lrng_hash_init(shash, hash)) {
+ returned_ent_bits = 0;
+ } else {
+ /*
+ * Do not truncate the output size exactly to collected_ent_bits
+ * as the aux pool may contain data that is not credited with
+ * entropy, but we want to use them to stir the DRNG state.
+ */
+ memcpy(outbuf, aux_output, requested_bits >> 3);
+ }
+
+ read_unlock_irqrestore(&drng->hash_lock, flags);
+ memzero_explicit(aux_output, digestsize);
+ return returned_ent_bits;
+}
+
+/* Fill the seed buffer with data from the noise sources */
+void lrng_fill_seed_buffer(struct entropy_buf *entropy_buf, u32 requested_bits)
+{
+ struct lrng_pool *pool = &lrng_pool;
+ struct lrng_state *state = &lrng_state;
+ unsigned long flags;
+ u32 pcpu_request, req_ent = lrng_sp80090c_compliant() ?
+ lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS;
+
+ /* Guarantee that requested bits is a multiple of bytes */
+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8);
+
+ /* always reseed the DRNG with the current time stamp */
+ entropy_buf->now = random_get_entropy();
+
+ /*
+ * Require at least 128 bits of entropy for any reseed. If the LRNG is
+ * operated SP800-90C compliant we want to comply with SP800-90A section
+ * 9.2 mandating that DRNG is reseeded with the security strength.
+ */
+ if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) {
+ entropy_buf->a_bits = entropy_buf->b_bits = 0;
+ entropy_buf->c_bits = entropy_buf->d_bits = 0;
+ goto wakeup;
+ }
+
+ /* Ensure aux pool extraction and backtracking op are atomic */
+ spin_lock_irqsave(&pool->lock, flags);
+
+ /* Concatenate the output of the entropy sources. */
+ entropy_buf->a_bits = lrng_get_aux_pool(entropy_buf->a, requested_bits);
+
+ /*
+ * If the aux pool returned entropy, pull respective less from per-CPU
+ * pool, but attempt to at least get LRNG_MIN_SEED_ENTROPY_BITS entropy.
+ */
+ pcpu_request = max_t(u32, requested_bits - entropy_buf->a_bits,
+ LRNG_MIN_SEED_ENTROPY_BITS);
+ entropy_buf->b_bits = lrng_pcpu_pool_hash(entropy_buf->b, pcpu_request,
+ state->lrng_fully_seeded);
+
+ entropy_buf->c_bits = lrng_get_arch(entropy_buf->c, requested_bits);
+ entropy_buf->d_bits = lrng_get_jent(entropy_buf->d, requested_bits);
+
+ /* Mix the extracted data back into pool for backtracking resistance */
+ if (lrng_pool_insert_aux_locked((u8 *)entropy_buf,
+ sizeof(struct entropy_buf), 0))
+ pr_warn("Backtracking resistance operation failed\n");
+
+ spin_unlock_irqrestore(&pool->lock, flags);
+
+ /* allow external entropy provider to provide seed */
+ lrng_state_exseed_allow_all();
+
+wakeup:
+ /*
+ * Shall we wake up user space writers? This location covers
+ * ensures that the user space provider does not dominate the internal
+ * noise sources since in case the first call of this function finds
+ * sufficient entropy in the entropy pool, it will not trigger the
+ * wakeup. This implies that when the next /dev/urandom read happens,
+ * the entropy pool is drained.
+ */
+ lrng_writer_wakeup();
+}
new file mode 100644
@@ -0,0 +1,702 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * LRNG Slow Entropy Source: Interrupt data collection
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <asm/irq_regs.h>
+#include <asm/ptrace.h>
+#include <crypto/hash.h>
+#include <linux/lrng.h>
+#include <linux/random.h>
+
+#include "lrng_internal.h"
+#include "lrng_sw_noise.h"
+
+/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */
+static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS;
+/* Is high-resolution timer present? */
+static bool lrng_irq_highres_timer = false;
+
+static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS;
+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG
+module_param(irq_entropy, uint, 0444);
+MODULE_PARM_DESC(irq_entropy,
+ "How many interrupts must be collected for obtaining 256 bits of entropy\n");
+#endif
+
+/* Per-CPU array holding concatenated entropy events */
+static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_pcpu_array)
+ __aligned(LRNG_KCAPI_ALIGN);
+static DEFINE_PER_CPU(u32, lrng_pcpu_array_ptr) = 0;
+static DEFINE_PER_CPU(atomic_t, lrng_pcpu_array_irqs) = ATOMIC_INIT(0);
+
+/*
+ * The entropy collection is performed by executing the following steps:
+ * 1. fill up the per-CPU array holding the time stamps
+ * 2. once the per-CPU array is full, a compression of the data into
+ * the entropy pool is performed - this happens in interrupt context
+ *
+ * If step 2 is not desired in interrupt context, the following boolean
+ * needs to be set to false. This implies that old entropy data in the
+ * per-CPU array collected since the last DRNG reseed is overwritten with
+ * new entropy data instead of retaining the entropy with the compression
+ * operation.
+ *
+ * Impact on entropy:
+ *
+ * If continuous compression is enabled, the maximum entropy that is collected
+ * per CPU between DRNG reseeds is equal to the digest size of the used hash.
+ *
+ * If continuous compression is disabled, the maximum number of entropy events
+ * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount
+ * of events is converted into an entropy statement which then represents the
+ * maximum amount of entropy collectible per CPU between DRNG reseeds.
+ */
+static bool lrng_pcpu_continuous_compression __read_mostly =
+ IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION);
+
+#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION
+module_param(lrng_pcpu_continuous_compression, bool, 0444);
+MODULE_PARM_DESC(lrng_pcpu_continuous_compression,
+ "Perform entropy compression if per-CPU entropy data array is full\n");
+#endif
+
+/*
+ * Per-CPU entropy pool with compressed entropy event
+ *
+ * The per-CPU entropy pool is defined as the hash state. New data is simply
+ * inserted into the entropy pool by performing a hash update operation.
+ * To read the entropy pool, a hash final must be invoked. However, before
+ * the entropy pool is released again after a hash final, the hash init must
+ * be performed.
+ */
+static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_pcpu_pool)
+ __aligned(LRNG_KCAPI_ALIGN);
+/*
+ * Lock to allow other CPUs to read the pool - as this is only done during
+ * reseed which is infrequent, this lock is hardly contended.
+ */
+static DEFINE_PER_CPU(spinlock_t, lrng_pcpu_lock);
+static DEFINE_PER_CPU(bool, lrng_pcpu_lock_init) = false;
+
+/* Return boolean whether LRNG identified presence of high-resolution timer */
+bool lrng_pool_highres_timer(void)
+{
+ return lrng_irq_highres_timer;
+}
+
+/* Convert entropy in bits into number of IRQs with the same entropy content. */
+static inline u32 lrng_entropy_to_data(u32 entropy_bits)
+{
+ return ((entropy_bits * lrng_irq_entropy_bits) /
+ LRNG_DRNG_SECURITY_STRENGTH_BITS);
+}
+
+/* Convert number of IRQs into entropy value. */
+static inline u32 lrng_data_to_entropy(u32 irqnum)
+{
+ return ((irqnum * LRNG_DRNG_SECURITY_STRENGTH_BITS) /
+ lrng_irq_entropy_bits);
+}
+
+static inline bool lrng_pcpu_pool_online(int cpu)
+{
+ return per_cpu(lrng_pcpu_lock_init, cpu);
+}
+
+bool lrng_pcpu_continuous_compression_state(void)
+{
+ return lrng_pcpu_continuous_compression;
+}
+
+static void lrng_pcpu_check_compression_state(void)
+{
+ /* One pool must hold sufficient entropy for disabled compression */
+ if (!lrng_pcpu_continuous_compression) {
+ u32 max_ent = min_t(u32, lrng_get_digestsize(),
+ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES));
+ if (max_ent < lrng_security_strength()) {
+ pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n");
+ lrng_pcpu_continuous_compression = true;
+ }
+ }
+}
+
+static int __init lrng_init_time_source(void)
+{
+ /* Set a minimum number of interrupts that must be collected */
+ irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy);
+
+ if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) ||
+ (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) {
+ /*
+ * As the highres timer is identified here, previous interrupts
+ * obtained during boot time are treated like a lowres-timer
+ * would have been present.
+ */
+ lrng_irq_highres_timer = true;
+ lrng_irq_entropy_bits = irq_entropy;
+ } else {
+ lrng_health_disable();
+ lrng_irq_highres_timer = false;
+ lrng_irq_entropy_bits = irq_entropy *
+ LRNG_IRQ_OVERSAMPLING_FACTOR;
+ pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n",
+ LRNG_IRQ_OVERSAMPLING_FACTOR);
+ lrng_pcpu_check_compression_state();
+ }
+
+ return 0;
+}
+core_initcall(lrng_init_time_source);
+
+/*
+ * Reset all per-CPU pools - reset entropy estimator but leave the pool data
+ * that may or may not have entropy unchanged.
+ */
+void lrng_pcpu_reset(void)
+{
+ int cpu;
+
+ for_each_online_cpu(cpu)
+ atomic_set(per_cpu_ptr(&lrng_pcpu_array_irqs, cpu), 0);
+}
+
+u32 lrng_pcpu_avail_pool_size(void)
+{
+ u32 max_size = 0, max_pool = lrng_get_digestsize();
+ int cpu;
+
+ if (!lrng_pcpu_continuous_compression)
+ max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES);
+
+ for_each_online_cpu(cpu) {
+ if (lrng_pcpu_pool_online(cpu))
+ max_size += max_pool;
+ }
+
+ return max_size;
+}
+
+/* Return entropy of unused IRQs present in all per-CPU pools. */
+u32 lrng_pcpu_avail_entropy(void)
+{
+ u32 digestsize_irqs, irq = 0;
+ int cpu;
+
+ /* Obtain the cap of maximum numbers of IRQs we count */
+ digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize());
+ if (!lrng_pcpu_continuous_compression) {
+ /* Cap to max. number of IRQs the array can hold */
+ digestsize_irqs = min_t(u32, digestsize_irqs,
+ LRNG_DATA_NUM_VALUES);
+ }
+
+ for_each_online_cpu(cpu) {
+ if (!lrng_pcpu_pool_online(cpu))
+ continue;
+ irq += min_t(u32, digestsize_irqs,
+ atomic_read_u32(per_cpu_ptr(&lrng_pcpu_array_irqs,
+ cpu)));
+ }
+
+ /* Consider oversampling rate */
+ return lrng_reduce_by_osr(lrng_data_to_entropy(irq));
+}
+
+/**
+ * Trigger a switch of the hash implementation for the per-CPU pool.
+ *
+ * For each per-CPU pool, obtain the message digest with the old hash
+ * implementation, initialize the per-CPU pool again with the new hash
+ * implementation and inject the message digest into the new state.
+ *
+ * Assumption: the caller must guarantee that the new_cb is available during the
+ * entire operation (e.g. it must hold the lock against pointer updating).
+ */
+int lrng_pcpu_switch_hash(int node,
+ const struct lrng_crypto_cb *new_cb, void *new_hash,
+ const struct lrng_crypto_cb *old_cb)
+{
+ u8 digest[LRNG_MAX_DIGESTSIZE];
+ u32 digestsize_irqs, found_irqs;
+ int ret = 0, cpu;
+
+ if (!IS_ENABLED(CONFIG_LRNG_DRNG_SWITCH))
+ return -EOPNOTSUPP;
+
+ for_each_online_cpu(cpu) {
+ struct shash_desc *pcpu_shash;
+
+ /*
+ * Only switch the per-CPU pools for the current node because
+ * the crypto_cb only applies NUMA-node-wide.
+ */
+ if (cpu_to_node(cpu) != node || !lrng_pcpu_pool_online(cpu))
+ continue;
+
+ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_pcpu_pool,
+ cpu);
+
+ digestsize_irqs = old_cb->lrng_hash_digestsize(pcpu_shash);
+ digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3);
+
+ if (pcpu_shash->tfm == new_hash)
+ continue;
+
+ /* Get the per-CPU pool hash with old digest ... */
+ ret = old_cb->lrng_hash_final(pcpu_shash, digest) ?:
+ /* ... re-initialize the hash with the new digest ... */
+ new_cb->lrng_hash_init(pcpu_shash, new_hash) ?:
+ /*
+ * ... feed the old hash into the new state. We may feed
+ * uninitialized memory into the new state, but this is
+ * considered no issue and even good as we have some more
+ * uncertainty here.
+ */
+ new_cb->lrng_hash_update(pcpu_shash, digest,
+ sizeof(digest));
+ if (ret)
+ goto out;
+
+ /*
+ * In case the new digest is larger than the old one, cap
+ * the available entropy to the old message digest used to
+ * process the existing data.
+ */
+ found_irqs = atomic_xchg_relaxed(
+ per_cpu_ptr(&lrng_pcpu_array_irqs, cpu), 0);
+ found_irqs = min_t(u32, found_irqs, digestsize_irqs);
+ atomic_add_return_relaxed(found_irqs,
+ per_cpu_ptr(&lrng_pcpu_array_irqs, cpu));
+
+ pr_debug("Re-initialize per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n",
+ cpu, node, new_cb->lrng_hash_name());
+ }
+
+out:
+ memzero_explicit(digest, sizeof(digest));
+ return ret;
+}
+
+/*
+ * When reading the per-CPU message digest, make sure we use the crypto
+ * callbacks defined for the NUMA node the per-CPU pool is defined for because
+ * the LRNG crypto switch support is only atomic per NUMA node.
+ */
+static inline u32
+lrng_pcpu_pool_hash_one(const struct lrng_crypto_cb *pcpu_crypto_cb,
+ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize)
+{
+ struct shash_desc *pcpu_shash =
+ (struct shash_desc *)per_cpu_ptr(lrng_pcpu_pool, cpu);
+ spinlock_t *lock = per_cpu_ptr(&lrng_pcpu_lock, cpu);
+ unsigned long flags;
+ u32 digestsize_irqs, found_irqs;
+
+ /* Lock guarding against reading / writing to per-CPU pool */
+ spin_lock_irqsave(lock, flags);
+
+ *digestsize = pcpu_crypto_cb->lrng_hash_digestsize(pcpu_hash);
+ digestsize_irqs = lrng_entropy_to_data(*digestsize << 3);
+
+ /* Obtain entropy statement like for the entropy pool */
+ found_irqs = atomic_xchg_relaxed(
+ per_cpu_ptr(&lrng_pcpu_array_irqs, cpu), 0);
+ /* Cap to maximum amount of data we can hold in hash */
+ found_irqs = min_t(u32, found_irqs, digestsize_irqs);
+
+ /* Cap to maximum amount of data we can hold in array */
+ if (!lrng_pcpu_continuous_compression)
+ found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES);
+
+ /* Store all not-yet compressed data in data array into hash, ... */
+ if (pcpu_crypto_cb->lrng_hash_update(pcpu_shash,
+ (u8 *)per_cpu_ptr(lrng_pcpu_array, cpu),
+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?:
+ /* ... get the per-CPU pool digest, ... */
+ pcpu_crypto_cb->lrng_hash_final(pcpu_shash, digest) ?:
+ /* ... re-initialize the hash, ... */
+ pcpu_crypto_cb->lrng_hash_init(pcpu_shash, pcpu_hash) ?:
+ /* ... feed the old hash into the new state. */
+ pcpu_crypto_cb->lrng_hash_update(pcpu_shash, digest, *digestsize))
+ found_irqs = 0;
+
+ spin_unlock_irqrestore(lock, flags);
+ return found_irqs;
+}
+
+/**
+ * Hash all per-CPU pools and return the digest to be used as seed data for
+ * seeding a DRNG. The caller must guarantee backtracking resistance.
+ * The function will only copy as much data as entropy is available into the
+ * caller-provided output buffer.
+ *
+ * This function handles the translation from the number of received interrupts
+ * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS
+ * which defines how many interrupts must be received to obtain 256 bits of
+ * entropy. With this value, the function lrng_data_to_entropy converts a given
+ * data size (received interrupts, requested amount of data, etc.) into an
+ * entropy statement. lrng_entropy_to_data does the reverse.
+ *
+ * @outbuf: buffer to store data in with size requested_bits
+ * @requested_bits: Requested amount of entropy
+ * @fully_seeded: indicator whether LRNG is fully seeded
+ * @return: amount of entropy in outbuf in bits.
+ */
+u32 lrng_pcpu_pool_hash(u8 *outbuf, u32 requested_bits, bool fully_seeded)
+{
+ SHASH_DESC_ON_STACK(shash, NULL);
+ const struct lrng_crypto_cb *crypto_cb;
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ struct lrng_drng *drng = lrng_drng_init_instance();
+ u8 digest[LRNG_MAX_DIGESTSIZE];
+ unsigned long flags, flags2;
+ u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs,
+ returned_ent_bits;
+ int ret, cpu;
+ void *hash;
+
+ /* Lock guarding replacement of per-NUMA hash */
+ read_lock_irqsave(&drng->hash_lock, flags);
+
+ crypto_cb = drng->crypto_cb;
+ hash = drng->hash;
+
+ /* The hash state of filled with all per-CPU pool hashes. */
+ ret = crypto_cb->lrng_hash_init(shash, hash);
+ if (ret)
+ goto err;
+
+ requested_irqs = lrng_entropy_to_data(requested_bits) +
+ lrng_compress_osr();
+
+ /*
+ * Harvest entropy from each per-CPU hash state - even though we may
+ * have collected sufficient entropy, we will hash all per-CPU pools.
+ */
+ for_each_online_cpu(cpu) {
+ struct lrng_drng *pcpu_drng = drng;
+ u32 digestsize, pcpu_unused_irqs = 0;
+ int node = cpu_to_node(cpu);
+
+ /* If pool is not online, then no entropy is present. */
+ if (!lrng_pcpu_pool_online(cpu))
+ continue;
+
+ if (lrng_drng && lrng_drng[node])
+ pcpu_drng = lrng_drng[node];
+
+ if (pcpu_drng == drng) {
+ found_irqs = lrng_pcpu_pool_hash_one(crypto_cb, hash,
+ cpu, digest,
+ &digestsize);
+ } else {
+ read_lock_irqsave(&pcpu_drng->hash_lock, flags2);
+ found_irqs =
+ lrng_pcpu_pool_hash_one(pcpu_drng->crypto_cb,
+ pcpu_drng->hash, cpu,
+ digest, &digestsize);
+ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2);
+ }
+
+ /* Inject the digest into the state of all per-CPU pools */
+ ret = crypto_cb->lrng_hash_update(shash, digest, digestsize);
+ if (ret)
+ goto err;
+
+ collected_irqs += found_irqs;
+ if (collected_irqs > requested_irqs) {
+ pcpu_unused_irqs = collected_irqs - requested_irqs;
+ atomic_add_return_relaxed(pcpu_unused_irqs,
+ per_cpu_ptr(&lrng_pcpu_array_irqs, cpu));
+ collected_irqs = requested_irqs;
+ }
+ pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n",
+ found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs);
+ }
+
+ ret = crypto_cb->lrng_hash_final(shash, digest);
+ if (ret)
+ goto err;
+
+ collected_ent_bits = lrng_data_to_entropy(collected_irqs);
+ /* Cap to maximum entropy that can ever be generated with given hash */
+ collected_ent_bits = min_t(u32, collected_ent_bits,
+ crypto_cb->lrng_hash_digestsize(hash) << 3);
+ /* Apply oversampling: discount requested oversampling rate */
+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits);
+
+ pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n",
+ returned_ent_bits, collected_ent_bits);
+
+ /*
+ * Truncate to available entropy as implicitly allowed by SP800-90B
+ * section 3.1.5.1.1 table 1 which awards truncated hashes full
+ * entropy.
+ *
+ * During boot time, we read requested_bits data with
+ * returned_ent_bits entropy. In case our conservative entropy
+ * estimate underestimates the available entropy we can transport as
+ * much available entropy as possible.
+ */
+ memcpy(outbuf, digest, fully_seeded ? returned_ent_bits >> 3 :
+ requested_bits >> 3);
+
+out:
+ crypto_cb->lrng_hash_desc_zero(shash);
+ read_unlock_irqrestore(&drng->hash_lock, flags);
+ memzero_explicit(digest, sizeof(digest));
+ return returned_ent_bits;
+
+err:
+ returned_ent_bits = 0;
+ goto out;
+}
+
+/* Compress the lrng_pcpu_array array into lrng_pcpu_pool */
+static inline void lrng_pcpu_array_compress(void)
+{
+ struct shash_desc *shash =
+ (struct shash_desc *)this_cpu_ptr(lrng_pcpu_pool);
+ struct lrng_drng **lrng_drng = lrng_drng_instances();
+ struct lrng_drng *drng = lrng_drng_init_instance();
+ const struct lrng_crypto_cb *crypto_cb;
+ spinlock_t *lock = this_cpu_ptr(&lrng_pcpu_lock);
+ unsigned long flags, flags2;
+ int node = numa_node_id();
+ void *hash;
+ bool init = false;
+
+ /* Get NUMA-node local hash instance */
+ if (lrng_drng && lrng_drng[node])
+ drng = lrng_drng[node];
+
+ read_lock_irqsave(&drng->hash_lock, flags);
+ crypto_cb = drng->crypto_cb;
+ hash = drng->hash;
+
+ if (unlikely(!this_cpu_read(lrng_pcpu_lock_init))) {
+ init = true;
+ spin_lock_init(lock);
+ this_cpu_write(lrng_pcpu_lock_init, true);
+ pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n",
+ raw_smp_processor_id(), node,
+ crypto_cb->lrng_hash_name());
+ }
+
+ spin_lock_irqsave(lock, flags2);
+
+ if (unlikely(init) && crypto_cb->lrng_hash_init(shash, hash)) {
+ this_cpu_write(lrng_pcpu_lock_init, false);
+ pr_warn("Initialization of hash failed\n");
+ } else if (lrng_pcpu_continuous_compression) {
+ /* Add entire per-CPU data array content into entropy pool. */
+ if (crypto_cb->lrng_hash_update(shash,
+ (u8 *)this_cpu_ptr(lrng_pcpu_array),
+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)))
+ pr_warn_ratelimited("Hashing of entropy data failed\n");
+ }
+
+ spin_unlock_irqrestore(lock, flags2);
+ read_unlock_irqrestore(&drng->hash_lock, flags);
+}
+
+/* Compress data array into hash */
+static inline void lrng_pcpu_array_to_hash(u32 ptr)
+{
+ u32 *array = this_cpu_ptr(lrng_pcpu_array);
+
+ /*
+ * During boot time the hash operation is triggered more often than
+ * during regular operation.
+ */
+ if (unlikely(!lrng_state_fully_seeded())) {
+ if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK))
+ return;
+ } else if (ptr < LRNG_DATA_WORD_MASK) {
+ return;
+ }
+
+ if (lrng_raw_array_entropy_store(*array)) {
+ u32 i;
+
+ /*
+ * If we fed even a part of the array to external analysis, we
+ * mark that the entire array and the per-CPU pool to have no
+ * entropy. This is due to the non-IID property of the data as
+ * we do not fully know whether the existing dependencies
+ * diminish the entropy beyond to what we expect it has.
+ */
+ atomic_set(this_cpu_ptr(&lrng_pcpu_array_irqs), 0);
+
+ for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++)
+ lrng_raw_array_entropy_store(*(array + i));
+ } else {
+ lrng_pcpu_array_compress();
+ /* Ping pool handler about received entropy */
+ lrng_pool_add_entropy();
+ }
+}
+
+/*
+ * Concatenate full 32 bit word at the end of time array even when current
+ * ptr is not aligned to sizeof(data).
+ */
+static inline void _lrng_pcpu_array_add_u32(u32 data)
+{
+ /* Increment pointer by number of slots taken for input value */
+ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_pcpu_array_ptr,
+ LRNG_DATA_SLOTS_PER_UINT);
+ unsigned int pre_array;
+
+ /*
+ * This function injects a unit into the array - guarantee that
+ * array unit size is equal to data type of input data.
+ */
+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3));
+
+ /*
+ * The following logic requires at least two units holding
+ * the data as otherwise the pointer would immediately wrap when
+ * injection an u32 word.
+ */
+ BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT);
+
+ lrng_pcpu_split_u32(&ptr, &pre_ptr, &mask);
+
+ /* MSB of data go into previous unit */
+ pre_array = lrng_data_idx2array(pre_ptr);
+ /* zeroization of slot to ensure the following OR adds the data */
+ this_cpu_and(lrng_pcpu_array[pre_array], ~(0xffffffff &~ mask));
+ this_cpu_or(lrng_pcpu_array[pre_array], data & ~mask);
+
+ /* Invoke compression as we just filled data array completely */
+ if (unlikely(pre_ptr > ptr))
+ lrng_pcpu_array_to_hash(LRNG_DATA_WORD_MASK);
+
+ /* LSB of data go into current unit */
+ this_cpu_write(lrng_pcpu_array[lrng_data_idx2array(ptr)],
+ data & mask);
+
+ if (likely(pre_ptr <= ptr))
+ lrng_pcpu_array_to_hash(ptr);
+}
+
+/* Concatenate a 32-bit word at the end of the per-CPU array */
+void lrng_pcpu_array_add_u32(u32 data)
+{
+ /*
+ * Disregard entropy-less data without continuous compression to
+ * avoid it overwriting data with entropy when array ptr wraps.
+ */
+ if (lrng_pcpu_continuous_compression)
+ _lrng_pcpu_array_add_u32(data);
+}
+
+/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */
+static inline void lrng_pcpu_array_add_slot(u32 data)
+{
+ /* Get slot */
+ u32 ptr = this_cpu_inc_return(lrng_pcpu_array_ptr) &
+ LRNG_DATA_WORD_MASK;
+ unsigned int array = lrng_data_idx2array(ptr);
+ unsigned int slot = lrng_data_idx2slot(ptr);
+
+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS);
+ /* Ensure consistency of values */
+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS !=
+ sizeof(lrng_pcpu_array[0]) << 3);
+
+ /* zeroization of slot to ensure the following OR adds the data */
+ this_cpu_and(lrng_pcpu_array[array],
+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK,
+ slot)));
+ /* Store data into slot */
+ this_cpu_or(lrng_pcpu_array[array], lrng_data_slot_val(data, slot));
+
+ lrng_pcpu_array_to_hash(ptr);
+}
+
+static inline void
+lrng_time_process_common(u32 time, void(*add_time)(u32 data))
+{
+ enum lrng_health_res health_test;
+
+ if (lrng_raw_hires_entropy_store(time))
+ return;
+
+ health_test = lrng_health_test(time);
+ if (health_test > lrng_health_fail_use)
+ return;
+
+ if (health_test == lrng_health_pass)
+ atomic_inc_return(this_cpu_ptr(&lrng_pcpu_array_irqs));
+
+ add_time(time);
+}
+
+/*
+ * Batching up of entropy in per-CPU array before injecting into entropy pool.
+ */
+static inline void lrng_time_process(void)
+{
+ u32 now_time = random_get_entropy();
+
+ if (unlikely(!lrng_state_fully_seeded())) {
+ /* During boot time, we process the full time stamp */
+ lrng_time_process_common(now_time, _lrng_pcpu_array_add_u32);
+ } else {
+ /* Runtime operation */
+ lrng_time_process_common(now_time & LRNG_DATA_SLOTSIZE_MASK,
+ lrng_pcpu_array_add_slot);
+ }
+
+ lrng_perf_time(now_time);
+}
+
+/* Hot code path - Callback for interrupt handler */
+void add_interrupt_randomness(int irq, int irq_flg)
+{
+ if (lrng_pool_highres_timer()) {
+ lrng_time_process();
+ } else {
+ struct pt_regs *regs = get_irq_regs();
+ static atomic_t reg_idx = ATOMIC_INIT(0);
+ u64 ip;
+ u32 tmp;
+
+ if (regs) {
+ u32 *ptr = (u32 *)regs;
+ int reg_ptr = atomic_add_return_relaxed(1, ®_idx);
+ size_t n = (sizeof(struct pt_regs) / sizeof(u32));
+
+ ip = instruction_pointer(regs);
+ tmp = *(ptr + (reg_ptr % n));
+ tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp;
+ _lrng_pcpu_array_add_u32(tmp);
+ } else {
+ ip = _RET_IP_;
+ }
+
+ lrng_time_process();
+
+ /*
+ * The XOR operation combining the different values is not
+ * considered to destroy entropy since the entirety of all
+ * processed values delivers the entropy (and not each
+ * value separately of the other values).
+ */
+ tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies;
+ tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq;
+ tmp ^= lrng_raw_irqflags_entropy_store(irq_flg) ? 0 : irq_flg;
+ tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip;
+ tmp ^= ip >> 32;
+ _lrng_pcpu_array_add_u32(tmp);
+ }
+}
+EXPORT_SYMBOL(add_interrupt_randomness);
new file mode 100644
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * LRNG Slow Noise Source: Time stamp array handling
+ *
+ * Copyright (C) 2016 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+/*
+ * To limit the impact on the interrupt handling, the LRNG concatenates
+ * entropic LSB parts of the time stamps in a per-CPU array and only
+ * injects them into the entropy pool when the array is full.
+ */
+
+/* Store multiple integers in one u32 */
+#define LRNG_DATA_SLOTSIZE_BITS (8)
+#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1)
+#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */
+#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \
+ LRNG_DATA_SLOTSIZE_BITS)
+
+/*
+ * Number of time values to store in the array - in small environments
+ * only one atomic_t variable per CPU is used.
+ */
+#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE)
+/* Mask of LSB of time stamp to store */
+#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1)
+
+#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1)
+#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \
+ LRNG_DATA_SLOTS_PER_UINT)
+
+/* Starting bit index of slot */
+static inline unsigned int lrng_data_slot2bitindex(unsigned int slot)
+{
+ return (LRNG_DATA_SLOTSIZE_BITS * slot);
+}
+
+/* Convert index into the array index */
+static inline unsigned int lrng_data_idx2array(unsigned int idx)
+{
+ return idx / LRNG_DATA_SLOTS_PER_UINT;
+}
+
+/* Convert index into the slot of a given array index */
+static inline unsigned int lrng_data_idx2slot(unsigned int idx)
+{
+ return idx & LRNG_DATA_SLOTS_MASK;
+}
+
+/* Convert value into slot value */
+static inline unsigned int lrng_data_slot_val(unsigned int val,
+ unsigned int slot)
+{
+ return val << lrng_data_slot2bitindex(slot);
+}
+
+/*
+ * Return the pointers for the previous and current units to inject a u32 into.
+ * Also return the mask which the u32 word is to be processed.
+ */
+static inline void lrng_pcpu_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask)
+{
+ /* ptr to previous unit */
+ *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK;
+ *ptr &= LRNG_DATA_WORD_MASK;
+
+ /* mask to split data into the two parts for the two units */
+ *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) *
+ LRNG_DATA_SLOTSIZE_BITS)) - 1;
+}
new file mode 100644
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (C) 2018 - 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#ifndef _LRNG_H
+#define _LRNG_H
+
+#include <crypto/hash.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+
+/**
+ * struct lrng_crypto_cb - cryptographic callback functions
+ * @lrng_drng_name Name of DRNG
+ * @lrng_hash_name Name of Hash used for reading entropy pool
+ * @lrng_drng_alloc: Allocate DRNG -- the provided integer should be
+ * used for sanity checks.
+ * return: allocated data structure or PTR_ERR on
+ * error
+ * @lrng_drng_dealloc: Deallocate DRNG
+ * @lrng_drng_seed_helper: Seed the DRNG with data of arbitrary length
+ * drng: is pointer to data structure allocated
+ * with lrng_drng_alloc
+ * return: >= 0 on success, < 0 on error
+ * @lrng_drng_generate_helper: Generate random numbers from the DRNG with
+ * arbitrary length
+ * @lrng_hash_alloc: Allocate the hash for reading the entropy pool
+ * return: allocated data structure (NULL is
+ * success too) or ERR_PTR on error
+ * @lrng_hash_dealloc: Deallocate Hash
+ * @lrng_hash_digestsize: Return the digestsize for the used hash to read
+ * out entropy pool
+ * hash: is pointer to data structure allocated
+ * with lrng_hash_alloc
+ * return: size of digest of hash in bytes
+ * @lrng_hash_init: Initialize hash
+ * hash: is pointer to data structure allocated
+ * with lrng_hash_alloc
+ * return: 0 on success, < 0 on error
+ * @lrng_hash_update: Update hash operation
+ * hash: is pointer to data structure allocated
+ * with lrng_hash_alloc
+ * return: 0 on success, < 0 on error
+ * @lrng_hash_final Final hash operation
+ * hash: is pointer to data structure allocated
+ * with lrng_hash_alloc
+ * return: 0 on success, < 0 on error
+ * @lrng_hash_desc_zero Zeroization of hash state buffer
+ *
+ * Assumptions:
+ *
+ * 1. Hash operation will not sleep
+ * 2. The hash' volatile state information is provided with *shash by caller.
+ */
+struct lrng_crypto_cb {
+ const char *(*lrng_drng_name)(void);
+ const char *(*lrng_hash_name)(void);
+ void *(*lrng_drng_alloc)(u32 sec_strength);
+ void (*lrng_drng_dealloc)(void *drng);
+ int (*lrng_drng_seed_helper)(void *drng, const u8 *inbuf, u32 inbuflen);
+ int (*lrng_drng_generate_helper)(void *drng, u8 *outbuf, u32 outbuflen);
+ void *(*lrng_hash_alloc)(void);
+ void (*lrng_hash_dealloc)(void *hash);
+ u32 (*lrng_hash_digestsize)(void *hash);
+ int (*lrng_hash_init)(struct shash_desc *shash, void *hash);
+ int (*lrng_hash_update)(struct shash_desc *shash, const u8 *inbuf,
+ u32 inbuflen);
+ int (*lrng_hash_final)(struct shash_desc *shash, u8 *digest);
+ void (*lrng_hash_desc_zero)(struct shash_desc *shash);
+};
+
+/* Register cryptographic backend */
+#ifdef CONFIG_LRNG_DRNG_SWITCH
+int lrng_set_drng_cb(const struct lrng_crypto_cb *cb);
+#else /* CONFIG_LRNG_DRNG_SWITCH */
+static inline int
+lrng_set_drng_cb(const struct lrng_crypto_cb *cb) { return -EOPNOTSUPP; }
+#endif /* CONFIG_LRNG_DRNG_SWITCH */
+
+#endif /* _LRNG_H */