1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Algorithm testing framework and tests.
4	*
5	* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
6	* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
7	* Copyright (c) 2007 Nokia Siemens Networks
8	* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
9	* Copyright (c) 2019 Google LLC
10	*
11	* Updated RFC4106 AES-GCM testing.
12	* Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
13	* Adrian Hoban <adrian.hoban@intel.com>
14	* Gabriele Paoloni <gabriele.paoloni@intel.com>
15	* Tadeusz Struk (tadeusz.struk@intel.com)
16	* Copyright (c) 2010, Intel Corporation.
17	*/
18
19	#include <crypto/aead.h>
20	#include <crypto/hash.h>
21	#include <crypto/skcipher.h>
22	#include <linux/err.h>
23	#include <linux/fips.h>
24	#include <linux/module.h>
25	#include <linux/once.h>
26	#include <linux/prandom.h>
27	#include <linux/scatterlist.h>
28	#include <linux/slab.h>
29	#include <linux/string.h>
30	#include <linux/uio.h>
31	#include <crypto/rng.h>
32	#include <crypto/drbg.h>
33	#include <crypto/akcipher.h>
34	#include <crypto/kpp.h>
35	#include <crypto/acompress.h>
36	#include <crypto/sig.h>
37	#include <crypto/internal/cipher.h>
38	#include <crypto/internal/simd.h>
39
40	#include "internal.h"
41
42	MODULE_IMPORT_NS("CRYPTO_INTERNAL");
43
44	static bool notests;
45	module_param(notests, bool, 0644);
46	MODULE_PARM_DESC(notests, "disable all crypto self-tests");
47
48	#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
49	static bool noslowtests;
50	module_param(noslowtests, bool, 0644);
51	MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests");
52
53	static unsigned int fuzz_iterations = 100;
54	module_param(fuzz_iterations, uint, 0644);
55	MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
56	#else
57	#define noslowtests 1
58	#define fuzz_iterations 0
59	#endif
60
61	#ifndef CONFIG_CRYPTO_SELFTESTS
62
63	/* a perfect nop */
64	int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
65	{
66	return 0;
67	}
68
69	#else
70
71	#include "testmgr.h"
72
73	/*
74	* Need slab memory for testing (size in number of pages).
75	*/
76	#define XBUFSIZE 8
77
78	/*
79	* Used by test_cipher()
80	*/
81	#define ENCRYPT 1
82	#define DECRYPT 0
83
84	struct aead_test_suite {
85	const struct aead_testvec *vecs;
86	unsigned int count;
87
88	/*
89	* Set if trying to decrypt an inauthentic ciphertext with this
90	* algorithm might result in EINVAL rather than EBADMSG, due to other
91	* validation the algorithm does on the inputs such as length checks.
92	*/
93	unsigned int einval_allowed : 1;
94
95	/*
96	* Set if this algorithm requires that the IV be located at the end of
97	* the AAD buffer, in addition to being given in the normal way. The
98	* behavior when the two IV copies differ is implementation-defined.
99	*/
100	unsigned int aad_iv : 1;
101	};
102
103	struct cipher_test_suite {
104	const struct cipher_testvec *vecs;
105	unsigned int count;
106	};
107
108	struct comp_test_suite {
109	struct {
110	const struct comp_testvec *vecs;
111	unsigned int count;
112	} comp, decomp;
113	};
114
115	struct hash_test_suite {
116	const struct hash_testvec *vecs;
117	unsigned int count;
118	};
119
120	struct drbg_test_suite {
121	const struct drbg_testvec *vecs;
122	unsigned int count;
123	};
124
125	struct akcipher_test_suite {
126	const struct akcipher_testvec *vecs;
127	unsigned int count;
128	};
129
130	struct sig_test_suite {
131	const struct sig_testvec *vecs;
132	unsigned int count;
133	};
134
135	struct kpp_test_suite {
136	const struct kpp_testvec *vecs;
137	unsigned int count;
138	};
139
140	struct alg_test_desc {
141	const char *alg;
142	const char *generic_driver;
143	int (*test)(const struct alg_test_desc *desc, const char *driver,
144	u32 type, u32 mask);
145	int fips_allowed; /* set if alg is allowed in fips mode */
146
147	union {
148	struct aead_test_suite aead;
149	struct cipher_test_suite cipher;
150	struct comp_test_suite comp;
151	struct hash_test_suite hash;
152	struct drbg_test_suite drbg;
153	struct akcipher_test_suite akcipher;
154	struct sig_test_suite sig;
155	struct kpp_test_suite kpp;
156	} suite;
157	};
158
159	static void hexdump(unsigned char *buf, unsigned int len)
160	{
161	print_hex_dump(KERN_CONT, prefix_str: "", prefix_type: DUMP_PREFIX_OFFSET,
162	rowsize: 16, groupsize: 1,
163	buf, len, ascii: false);
164	}
165
166	static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
167	{
168	int i;
169
170	for (i = 0; i < XBUFSIZE; i++) {
171	buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
172	if (!buf[i])
173	goto err_free_buf;
174	}
175
176	return 0;
177
178	err_free_buf:
179	while (i-- > 0)
180	free_pages(addr: (unsigned long)buf[i], order);
181
182	return -ENOMEM;
183	}
184
185	static int testmgr_alloc_buf(char *buf[XBUFSIZE])
186	{
187	return __testmgr_alloc_buf(buf, order: 0);
188	}
189
190	static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
191	{
192	int i;
193
194	for (i = 0; i < XBUFSIZE; i++)
195	free_pages(addr: (unsigned long)buf[i], order);
196	}
197
198	static void testmgr_free_buf(char *buf[XBUFSIZE])
199	{
200	__testmgr_free_buf(buf, order: 0);
201	}
202
203	#define TESTMGR_POISON_BYTE 0xfe
204	#define TESTMGR_POISON_LEN 16
205
206	static inline void testmgr_poison(void *addr, size_t len)
207	{
208	memset(addr, TESTMGR_POISON_BYTE, len);
209	}
210
211	/* Is the memory region still fully poisoned? */
212	static inline bool testmgr_is_poison(const void *addr, size_t len)
213	{
214	return memchr_inv(p: addr, TESTMGR_POISON_BYTE, size: len) == NULL;
215	}
216
217	/* flush type for hash algorithms */
218	enum flush_type {
219	/* merge with update of previous buffer(s) */
220	FLUSH_TYPE_NONE = 0,
221
222	/* update with previous buffer(s) before doing this one */
223	FLUSH_TYPE_FLUSH,
224
225	/* likewise, but also export and re-import the intermediate state */
226	FLUSH_TYPE_REIMPORT,
227	};
228
229	/* finalization function for hash algorithms */
230	enum finalization_type {
231	FINALIZATION_TYPE_FINAL, /* use final() */
232	FINALIZATION_TYPE_FINUP, /* use finup() */
233	FINALIZATION_TYPE_DIGEST, /* use digest() */
234	};
235
236	/*
237	* Whether the crypto operation will occur in-place, and if so whether the
238	* source and destination scatterlist pointers will coincide (req->src ==
239	* req->dst), or whether they'll merely point to two separate scatterlists
240	* (req->src != req->dst) that reference the same underlying memory.
241	*
242	* This is only relevant for algorithm types that support in-place operation.
243	*/
244	enum inplace_mode {
245	OUT_OF_PLACE,
246	INPLACE_ONE_SGLIST,
247	INPLACE_TWO_SGLISTS,
248	};
249
250	#define TEST_SG_TOTAL 10000
251
252	/**
253	* struct test_sg_division - description of a scatterlist entry
254	*
255	* This struct describes one entry of a scatterlist being constructed to check a
256	* crypto test vector.
257	*
258	* @proportion_of_total: length of this chunk relative to the total length,
259	* given as a proportion out of TEST_SG_TOTAL so that it
260	* scales to fit any test vector
261	* @offset: byte offset into a 2-page buffer at which this chunk will start
262	* @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
263	* @offset
264	* @flush_type: for hashes, whether an update() should be done now vs.
265	* continuing to accumulate data
266	* @nosimd: if doing the pending update(), do it with SIMD disabled?
267	*/
268	struct test_sg_division {
269	unsigned int proportion_of_total;
270	unsigned int offset;
271	bool offset_relative_to_alignmask;
272	enum flush_type flush_type;
273	bool nosimd;
274	};
275
276	/**
277	* struct testvec_config - configuration for testing a crypto test vector
278	*
279	* This struct describes the data layout and other parameters with which each
280	* crypto test vector can be tested.
281	*
282	* @name: name of this config, logged for debugging purposes if a test fails
283	* @inplace_mode: whether and how to operate on the data in-place, if applicable
284	* @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
285	* @src_divs: description of how to arrange the source scatterlist
286	* @dst_divs: description of how to arrange the dst scatterlist, if applicable
287	* for the algorithm type. Defaults to @src_divs if unset.
288	* @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
289	* where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
290	* @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
291	* the @iv_offset
292	* @key_offset: misalignment of the key, where 0 is default alignment
293	* @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
294	* the @key_offset
295	* @finalization_type: what finalization function to use for hashes
296	* @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
297	* This applies to the parts of the operation that aren't controlled
298	* individually by @nosimd_setkey or @src_divs[].nosimd.
299	* @nosimd_setkey: set the key (if applicable) with SIMD disabled? Requires
300	* !CRYPTO_TFM_REQ_MAY_SLEEP.
301	*/
302	struct testvec_config {
303	const char *name;
304	enum inplace_mode inplace_mode;
305	u32 req_flags;
306	struct test_sg_division src_divs[XBUFSIZE];
307	struct test_sg_division dst_divs[XBUFSIZE];
308	unsigned int iv_offset;
309	unsigned int key_offset;
310	bool iv_offset_relative_to_alignmask;
311	bool key_offset_relative_to_alignmask;
312	enum finalization_type finalization_type;
313	bool nosimd;
314	bool nosimd_setkey;
315	};
316
317	#define TESTVEC_CONFIG_NAMELEN 192
318
319	/*
320	* The following are the lists of testvec_configs to test for each algorithm
321	* type when the "fast" crypto self-tests are enabled. They aim to provide good
322	* test coverage, while keeping the test time much shorter than the "full" tests
323	* so that the "fast" tests can be enabled in a wider range of circumstances.
324	*/
325
326	/* Configs for skciphers and aeads */
327	static const struct testvec_config default_cipher_testvec_configs[] = {
328	{
329	.name = "in-place (one sglist)",
330	.inplace_mode = INPLACE_ONE_SGLIST,
331	.src_divs = { { .proportion_of_total = 10000 } },
332	}, {
333	.name = "in-place (two sglists)",
334	.inplace_mode = INPLACE_TWO_SGLISTS,
335	.src_divs = { { .proportion_of_total = 10000 } },
336	}, {
337	.name = "out-of-place",
338	.inplace_mode = OUT_OF_PLACE,
339	.src_divs = { { .proportion_of_total = 10000 } },
340	}, {
341	.name = "unaligned buffer, offset=1",
342	.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
343	.iv_offset = 1,
344	.key_offset = 1,
345	}, {
346	.name = "buffer aligned only to alignmask",
347	.src_divs = {
348	{
349	.proportion_of_total = 10000,
350	.offset = 1,
351	.offset_relative_to_alignmask = true,
352	},
353	},
354	.iv_offset = 1,
355	.iv_offset_relative_to_alignmask = true,
356	.key_offset = 1,
357	.key_offset_relative_to_alignmask = true,
358	}, {
359	.name = "two even aligned splits",
360	.src_divs = {
361	{ .proportion_of_total = 5000 },
362	{ .proportion_of_total = 5000 },
363	},
364	}, {
365	.name = "one src, two even splits dst",
366	.inplace_mode = OUT_OF_PLACE,
367	.src_divs = { { .proportion_of_total = 10000 } },
368	.dst_divs = {
369	{ .proportion_of_total = 5000 },
370	{ .proportion_of_total = 5000 },
371	},
372	}, {
373	.name = "uneven misaligned splits, may sleep",
374	.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
375	.src_divs = {
376	{ .proportion_of_total = 1900, .offset = 33 },
377	{ .proportion_of_total = 3300, .offset = 7 },
378	{ .proportion_of_total = 4800, .offset = 18 },
379	},
380	.iv_offset = 3,
381	.key_offset = 3,
382	}, {
383	.name = "misaligned splits crossing pages, inplace",
384	.inplace_mode = INPLACE_ONE_SGLIST,
385	.src_divs = {
386	{
387	.proportion_of_total = 7500,
388	.offset = PAGE_SIZE - 32
389	}, {
390	.proportion_of_total = 2500,
391	.offset = PAGE_SIZE - 7
392	},
393	},
394	}
395	};
396
397	static const struct testvec_config default_hash_testvec_configs[] = {
398	{
399	.name = "init+update+final aligned buffer",
400	.src_divs = { { .proportion_of_total = 10000 } },
401	.finalization_type = FINALIZATION_TYPE_FINAL,
402	}, {
403	.name = "init+finup aligned buffer",
404	.src_divs = { { .proportion_of_total = 10000 } },
405	.finalization_type = FINALIZATION_TYPE_FINUP,
406	}, {
407	.name = "digest aligned buffer",
408	.src_divs = { { .proportion_of_total = 10000 } },
409	.finalization_type = FINALIZATION_TYPE_DIGEST,
410	}, {
411	.name = "init+update+final misaligned buffer",
412	.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
413	.finalization_type = FINALIZATION_TYPE_FINAL,
414	.key_offset = 1,
415	}, {
416	.name = "digest misaligned buffer",
417	.src_divs = {
418	{
419	.proportion_of_total = 10000,
420	.offset = 1,
421	},
422	},
423	.finalization_type = FINALIZATION_TYPE_DIGEST,
424	.key_offset = 1,
425	}, {
426	.name = "init+update+update+final two even splits",
427	.src_divs = {
428	{ .proportion_of_total = 5000 },
429	{
430	.proportion_of_total = 5000,
431	.flush_type = FLUSH_TYPE_FLUSH,
432	},
433	},
434	.finalization_type = FINALIZATION_TYPE_FINAL,
435	}, {
436	.name = "digest uneven misaligned splits, may sleep",
437	.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
438	.src_divs = {
439	{ .proportion_of_total = 1900, .offset = 33 },
440	{ .proportion_of_total = 3300, .offset = 7 },
441	{ .proportion_of_total = 4800, .offset = 18 },
442	},
443	.finalization_type = FINALIZATION_TYPE_DIGEST,
444	}, {
445	.name = "digest misaligned splits crossing pages",
446	.src_divs = {
447	{
448	.proportion_of_total = 7500,
449	.offset = PAGE_SIZE - 32,
450	}, {
451	.proportion_of_total = 2500,
452	.offset = PAGE_SIZE - 7,
453	},
454	},
455	.finalization_type = FINALIZATION_TYPE_DIGEST,
456	}, {
457	.name = "import/export",
458	.src_divs = {
459	{
460	.proportion_of_total = 6500,
461	.flush_type = FLUSH_TYPE_REIMPORT,
462	}, {
463	.proportion_of_total = 3500,
464	.flush_type = FLUSH_TYPE_REIMPORT,
465	},
466	},
467	.finalization_type = FINALIZATION_TYPE_FINAL,
468	}
469	};
470
471	static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
472	{
473	unsigned int remaining = TEST_SG_TOTAL;
474	unsigned int ndivs = 0;
475
476	do {
477	remaining -= divs[ndivs++].proportion_of_total;
478	} while (remaining);
479
480	return ndivs;
481	}
482
483	#define SGDIVS_HAVE_FLUSHES BIT(0)
484	#define SGDIVS_HAVE_NOSIMD BIT(1)
485
486	static bool valid_sg_divisions(const struct test_sg_division *divs,
487	unsigned int count, int *flags_ret)
488	{
489	unsigned int total = 0;
490	unsigned int i;
491
492	for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
493	if (divs[i].proportion_of_total <= 0 ||
494	divs[i].proportion_of_total > TEST_SG_TOTAL - total)
495	return false;
496	total += divs[i].proportion_of_total;
497	if (divs[i].flush_type != FLUSH_TYPE_NONE)
498	*flags_ret |= SGDIVS_HAVE_FLUSHES;
499	if (divs[i].nosimd)
500	*flags_ret |= SGDIVS_HAVE_NOSIMD;
501	}
502	return total == TEST_SG_TOTAL &&
503	memchr_inv(p: &divs[i], c: 0, size: (count - i) * sizeof(divs[0])) == NULL;
504	}
505
506	/*
507	* Check whether the given testvec_config is valid. This isn't strictly needed
508	* since every testvec_config should be valid, but check anyway so that people
509	* don't unknowingly add broken configs that don't do what they wanted.
510	*/
511	static bool valid_testvec_config(const struct testvec_config *cfg)
512	{
513	int flags = 0;
514
515	if (cfg->name == NULL)
516	return false;
517
518	if (!valid_sg_divisions(divs: cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
519	flags_ret: &flags))
520	return false;
521
522	if (cfg->dst_divs[0].proportion_of_total) {
523	if (!valid_sg_divisions(divs: cfg->dst_divs,
524	ARRAY_SIZE(cfg->dst_divs), flags_ret: &flags))
525	return false;
526	} else {
527	if (memchr_inv(p: cfg->dst_divs, c: 0, size: sizeof(cfg->dst_divs)))
528	return false;
529	/* defaults to dst_divs=src_divs */
530	}
531
532	if (cfg->iv_offset +
533	(cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
534	MAX_ALGAPI_ALIGNMASK + 1)
535	return false;
536
537	if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
538	cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
539	return false;
540
541	if ((cfg->nosimd || cfg->nosimd_setkey ||
542	(flags & SGDIVS_HAVE_NOSIMD)) &&
543	(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
544	return false;
545
546	return true;
547	}
548
549	struct test_sglist {
550	char *bufs[XBUFSIZE];
551	struct scatterlist sgl[XBUFSIZE];
552	struct scatterlist sgl_saved[XBUFSIZE];
553	struct scatterlist *sgl_ptr;
554	unsigned int nents;
555	};
556
557	static int init_test_sglist(struct test_sglist *tsgl)
558	{
559	return __testmgr_alloc_buf(buf: tsgl->bufs, order: 1 /* two pages per buffer */);
560	}
561
562	static void destroy_test_sglist(struct test_sglist *tsgl)
563	{
564	return __testmgr_free_buf(buf: tsgl->bufs, order: 1 /* two pages per buffer */);
565	}
566
567	/**
568	* build_test_sglist() - build a scatterlist for a crypto test
569	*
570	* @tsgl: the scatterlist to build. @tsgl->bufs[] contains an array of 2-page
571	* buffers which the scatterlist @tsgl->sgl[] will be made to point into.
572	* @divs: the layout specification on which the scatterlist will be based
573	* @alignmask: the algorithm's alignmask
574	* @total_len: the total length of the scatterlist to build in bytes
575	* @data: if non-NULL, the buffers will be filled with this data until it ends.
576	* Otherwise the buffers will be poisoned. In both cases, some bytes
577	* past the end of each buffer will be poisoned to help detect overruns.
578	* @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
579	* corresponds will be returned here. This will match @divs except
580	* that divisions resolving to a length of 0 are omitted as they are
581	* not included in the scatterlist.
582	*
583	* Return: 0 or a -errno value
584	*/
585	static int build_test_sglist(struct test_sglist *tsgl,
586	const struct test_sg_division *divs,
587	const unsigned int alignmask,
588	const unsigned int total_len,
589	struct iov_iter *data,
590	const struct test_sg_division *out_divs[XBUFSIZE])
591	{
592	struct {
593	const struct test_sg_division *div;
594	size_t length;
595	} partitions[XBUFSIZE];
596	const unsigned int ndivs = count_test_sg_divisions(divs);
597	unsigned int len_remaining = total_len;
598	unsigned int i;
599
600	BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
601	if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
602	return -EINVAL;
603
604	/* Calculate the (div, length) pairs */
605	tsgl->nents = 0;
606	for (i = 0; i < ndivs; i++) {
607	unsigned int len_this_sg =
608	min(len_remaining,
609	(total_len * divs[i].proportion_of_total +
610	TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
611
612	if (len_this_sg != 0) {
613	partitions[tsgl->nents].div = &divs[i];
614	partitions[tsgl->nents].length = len_this_sg;
615	tsgl->nents++;
616	len_remaining -= len_this_sg;
617	}
618	}
619	if (tsgl->nents == 0) {
620	partitions[tsgl->nents].div = &divs[0];
621	partitions[tsgl->nents].length = 0;
622	tsgl->nents++;
623	}
624	partitions[tsgl->nents - 1].length += len_remaining;
625
626	/* Set up the sgl entries and fill the data or poison */
627	sg_init_table(tsgl->sgl, tsgl->nents);
628	for (i = 0; i < tsgl->nents; i++) {
629	unsigned int offset = partitions[i].div->offset;
630	void *addr;
631
632	if (partitions[i].div->offset_relative_to_alignmask)
633	offset += alignmask;
634
635	while (offset + partitions[i].length + TESTMGR_POISON_LEN >
636	2 * PAGE_SIZE) {
637	if (WARN_ON(offset <= 0))
638	return -EINVAL;
639	offset /= 2;
640	}
641
642	addr = &tsgl->bufs[i][offset];
643	sg_set_buf(sg: &tsgl->sgl[i], buf: addr, buflen: partitions[i].length);
644
645	if (out_divs)
646	out_divs[i] = partitions[i].div;
647
648	if (data) {
649	size_t copy_len, copied;
650
651	copy_len = min(partitions[i].length, data->count);
652	copied = copy_from_iter(addr, bytes: copy_len, i: data);
653	if (WARN_ON(copied != copy_len))
654	return -EINVAL;
655	testmgr_poison(addr: addr + copy_len, len: partitions[i].length +
656	TESTMGR_POISON_LEN - copy_len);
657	} else {
658	testmgr_poison(addr, len: partitions[i].length +
659	TESTMGR_POISON_LEN);
660	}
661	}
662
663	sg_mark_end(sg: &tsgl->sgl[tsgl->nents - 1]);
664	tsgl->sgl_ptr = tsgl->sgl;
665	memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
666	return 0;
667	}
668
669	/*
670	* Verify that a scatterlist crypto operation produced the correct output.
671	*
672	* @tsgl: scatterlist containing the actual output
673	* @expected_output: buffer containing the expected output
674	* @len_to_check: length of @expected_output in bytes
675	* @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
676	* @check_poison: verify that the poison bytes after each chunk are intact?
677	*
678	* Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
679	*/
680	static int verify_correct_output(const struct test_sglist *tsgl,
681	const char *expected_output,
682	unsigned int len_to_check,
683	unsigned int unchecked_prefix_len,
684	bool check_poison)
685	{
686	unsigned int i;
687
688	for (i = 0; i < tsgl->nents; i++) {
689	struct scatterlist *sg = &tsgl->sgl_ptr[i];
690	unsigned int len = sg->length;
691	unsigned int offset = sg->offset;
692	const char *actual_output;
693
694	if (unchecked_prefix_len) {
695	if (unchecked_prefix_len >= len) {
696	unchecked_prefix_len -= len;
697	continue;
698	}
699	offset += unchecked_prefix_len;
700	len -= unchecked_prefix_len;
701	unchecked_prefix_len = 0;
702	}
703	len = min(len, len_to_check);
704	actual_output = page_address(sg_page(sg)) + offset;
705	if (memcmp(p: expected_output, q: actual_output, size: len) != 0)
706	return -EINVAL;
707	if (check_poison &&
708	!testmgr_is_poison(addr: actual_output + len, TESTMGR_POISON_LEN))
709	return -EOVERFLOW;
710	len_to_check -= len;
711	expected_output += len;
712	}
713	if (WARN_ON(len_to_check != 0))
714	return -EINVAL;
715	return 0;
716	}
717
718	static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
719	{
720	unsigned int i;
721
722	for (i = 0; i < tsgl->nents; i++) {
723	if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
724	return true;
725	if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
726	return true;
727	if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
728	return true;
729	}
730	return false;
731	}
732
733	struct cipher_test_sglists {
734	struct test_sglist src;
735	struct test_sglist dst;
736	};
737
738	static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
739	{
740	struct cipher_test_sglists *tsgls;
741
742	tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
743	if (!tsgls)
744	return NULL;
745
746	if (init_test_sglist(tsgl: &tsgls->src) != 0)
747	goto fail_kfree;
748	if (init_test_sglist(tsgl: &tsgls->dst) != 0)
749	goto fail_destroy_src;
750
751	return tsgls;
752
753	fail_destroy_src:
754	destroy_test_sglist(tsgl: &tsgls->src);
755	fail_kfree:
756	kfree(objp: tsgls);
757	return NULL;
758	}
759
760	static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
761	{
762	if (tsgls) {
763	destroy_test_sglist(tsgl: &tsgls->src);
764	destroy_test_sglist(tsgl: &tsgls->dst);
765	kfree(objp: tsgls);
766	}
767	}
768
769	/* Build the src and dst scatterlists for an skcipher or AEAD test */
770	static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
771	const struct testvec_config *cfg,
772	unsigned int alignmask,
773	unsigned int src_total_len,
774	unsigned int dst_total_len,
775	const struct kvec *inputs,
776	unsigned int nr_inputs)
777	{
778	struct iov_iter input;
779	int err;
780
781	iov_iter_kvec(i: &input, ITER_SOURCE, kvec: inputs, nr_segs: nr_inputs, count: src_total_len);
782	err = build_test_sglist(tsgl: &tsgls->src, divs: cfg->src_divs, alignmask,
783	total_len: cfg->inplace_mode != OUT_OF_PLACE ?
784	max(dst_total_len, src_total_len) :
785	src_total_len,
786	data: &input, NULL);
787	if (err)
788	return err;
789
790	/*
791	* In-place crypto operations can use the same scatterlist for both the
792	* source and destination (req->src == req->dst), or can use separate
793	* scatterlists (req->src != req->dst) which point to the same
794	* underlying memory. Make sure to test both cases.
795	*/
796	if (cfg->inplace_mode == INPLACE_ONE_SGLIST) {
797	tsgls->dst.sgl_ptr = tsgls->src.sgl;
798	tsgls->dst.nents = tsgls->src.nents;
799	return 0;
800	}
801	if (cfg->inplace_mode == INPLACE_TWO_SGLISTS) {
802	/*
803	* For now we keep it simple and only test the case where the
804	* two scatterlists have identical entries, rather than
805	* different entries that split up the same memory differently.
806	*/
807	memcpy(tsgls->dst.sgl, tsgls->src.sgl,
808	tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
809	memcpy(tsgls->dst.sgl_saved, tsgls->src.sgl,
810	tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
811	tsgls->dst.sgl_ptr = tsgls->dst.sgl;
812	tsgls->dst.nents = tsgls->src.nents;
813	return 0;
814	}
815	/* Out of place */
816	return build_test_sglist(tsgl: &tsgls->dst,
817	divs: cfg->dst_divs[0].proportion_of_total ?
818	cfg->dst_divs : cfg->src_divs,
819	alignmask, total_len: dst_total_len, NULL, NULL);
820	}
821
822	/*
823	* Support for testing passing a misaligned key to setkey():
824	*
825	* If cfg->key_offset is set, copy the key into a new buffer at that offset,
826	* optionally adding alignmask. Else, just use the key directly.
827	*/
828	static int prepare_keybuf(const u8 *key, unsigned int ksize,
829	const struct testvec_config *cfg,
830	unsigned int alignmask,
831	const u8 **keybuf_ret, const u8 **keyptr_ret)
832	{
833	unsigned int key_offset = cfg->key_offset;
834	u8 *keybuf = NULL, *keyptr = (u8 *)key;
835
836	if (key_offset != 0) {
837	if (cfg->key_offset_relative_to_alignmask)
838	key_offset += alignmask;
839	keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
840	if (!keybuf)
841	return -ENOMEM;
842	keyptr = keybuf + key_offset;
843	memcpy(keyptr, key, ksize);
844	}
845	*keybuf_ret = keybuf;
846	*keyptr_ret = keyptr;
847	return 0;
848	}
849
850	/*
851	* Like setkey_f(tfm, key, ksize), but sometimes misalign the key.
852	* In addition, run the setkey function in no-SIMD context if requested.
853	*/
854	#define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask) \
855	({ \
856	const u8 *keybuf, *keyptr; \
857	int err; \
858	\
859	err = prepare_keybuf((key), (ksize), (cfg), (alignmask), \
860	&keybuf, &keyptr); \
861	if (err == 0) { \
862	if ((cfg)->nosimd_setkey) \
863	crypto_disable_simd_for_test(); \
864	err = setkey_f((tfm), keyptr, (ksize)); \
865	if ((cfg)->nosimd_setkey) \
866	crypto_reenable_simd_for_test(); \
867	kfree(keybuf); \
868	} \
869	err; \
870	})
871
872	/*
873	* The fuzz tests use prandom instead of the normal Linux RNG since they don't
874	* need cryptographically secure random numbers. This greatly improves the
875	* performance of these tests, especially if they are run before the Linux RNG
876	* has been initialized or if they are run on a lockdep-enabled kernel.
877	*/
878
879	static inline void init_rnd_state(struct rnd_state *rng)
880	{
881	prandom_seed_state(state: rng, seed: get_random_u64());
882	}
883
884	static inline u8 prandom_u8(struct rnd_state *rng)
885	{
886	return prandom_u32_state(state: rng);
887	}
888
889	static inline u32 prandom_u32_below(struct rnd_state *rng, u32 ceil)
890	{
891	/*
892	* This is slightly biased for non-power-of-2 values of 'ceil', but this
893	* isn't important here.
894	*/
895	return prandom_u32_state(state: rng) % ceil;
896	}
897
898	static inline bool prandom_bool(struct rnd_state *rng)
899	{
900	return prandom_u32_below(rng, ceil: 2);
901	}
902
903	static inline u32 prandom_u32_inclusive(struct rnd_state *rng,
904	u32 floor, u32 ceil)
905	{
906	return floor + prandom_u32_below(rng, ceil: ceil - floor + 1);
907	}
908
909	/* Generate a random length in range [0, max_len], but prefer smaller values */
910	static unsigned int generate_random_length(struct rnd_state *rng,
911	unsigned int max_len)
912	{
913	unsigned int len = prandom_u32_below(rng, ceil: max_len + 1);
914
915	switch (prandom_u32_below(rng, ceil: 4)) {
916	case 0:
917	len %= 64;
918	break;
919	case 1:
920	len %= 256;
921	break;
922	case 2:
923	len %= 1024;
924	break;
925	default:
926	break;
927	}
928	if (len && prandom_u32_below(rng, ceil: 4) == 0)
929	len = rounddown_pow_of_two(len);
930	return len;
931	}
932
933	/* Flip a random bit in the given nonempty data buffer */
934	static void flip_random_bit(struct rnd_state *rng, u8 *buf, size_t size)
935	{
936	size_t bitpos;
937
938	bitpos = prandom_u32_below(rng, ceil: size * 8);
939	buf[bitpos / 8] ^= 1 << (bitpos % 8);
940	}
941
942	/* Flip a random byte in the given nonempty data buffer */
943	static void flip_random_byte(struct rnd_state *rng, u8 *buf, size_t size)
944	{
945	buf[prandom_u32_below(rng, ceil: size)] ^= 0xff;
946	}
947
948	/* Sometimes make some random changes to the given nonempty data buffer */
949	static void mutate_buffer(struct rnd_state *rng, u8 *buf, size_t size)
950	{
951	size_t num_flips;
952	size_t i;
953
954	/* Sometimes flip some bits */
955	if (prandom_u32_below(rng, ceil: 4) == 0) {
956	num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8),
957	size * 8);
958	for (i = 0; i < num_flips; i++)
959	flip_random_bit(rng, buf, size);
960	}
961
962	/* Sometimes flip some bytes */
963	if (prandom_u32_below(rng, ceil: 4) == 0) {
964	num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8), size);
965	for (i = 0; i < num_flips; i++)
966	flip_random_byte(rng, buf, size);
967	}
968	}
969
970	/* Randomly generate 'count' bytes, but sometimes make them "interesting" */
971	static void generate_random_bytes(struct rnd_state *rng, u8 *buf, size_t count)
972	{
973	u8 b;
974	u8 increment;
975	size_t i;
976
977	if (count == 0)
978	return;
979
980	switch (prandom_u32_below(rng, ceil: 8)) { /* Choose a generation strategy */
981	case 0:
982	case 1:
983	/* All the same byte, plus optional mutations */
984	switch (prandom_u32_below(rng, ceil: 4)) {
985	case 0:
986	b = 0x00;
987	break;
988	case 1:
989	b = 0xff;
990	break;
991	default:
992	b = prandom_u8(rng);
993	break;
994	}
995	memset(buf, b, count);
996	mutate_buffer(rng, buf, size: count);
997	break;
998	case 2:
999	/* Ascending or descending bytes, plus optional mutations */
1000	increment = prandom_u8(rng);
1001	b = prandom_u8(rng);
1002	for (i = 0; i < count; i++, b += increment)
1003	buf[i] = b;
1004	mutate_buffer(rng, buf, size: count);
1005	break;
1006	default:
1007	/* Fully random bytes */
1008	prandom_bytes_state(state: rng, buf, nbytes: count);
1009	}
1010	}
1011
1012	static char *generate_random_sgl_divisions(struct rnd_state *rng,
1013	struct test_sg_division *divs,
1014	size_t max_divs, char *p, char *end,
1015	bool gen_flushes, u32 req_flags)
1016	{
1017	struct test_sg_division *div = divs;
1018	unsigned int remaining = TEST_SG_TOTAL;
1019
1020	do {
1021	unsigned int this_len;
1022	const char *flushtype_str;
1023
1024	if (div == &divs[max_divs - 1] || prandom_bool(rng))
1025	this_len = remaining;
1026	else if (prandom_u32_below(rng, ceil: 4) == 0)
1027	this_len = (remaining + 1) / 2;
1028	else
1029	this_len = prandom_u32_inclusive(rng, floor: 1, ceil: remaining);
1030	div->proportion_of_total = this_len;
1031
1032	if (prandom_u32_below(rng, ceil: 4) == 0)
1033	div->offset = prandom_u32_inclusive(rng,
1034	PAGE_SIZE - 128,
1035	PAGE_SIZE - 1);
1036	else if (prandom_bool(rng))
1037	div->offset = prandom_u32_below(rng, ceil: 32);
1038	else
1039	div->offset = prandom_u32_below(rng, PAGE_SIZE);
1040	if (prandom_u32_below(rng, ceil: 8) == 0)
1041	div->offset_relative_to_alignmask = true;
1042
1043	div->flush_type = FLUSH_TYPE_NONE;
1044	if (gen_flushes) {
1045	switch (prandom_u32_below(rng, ceil: 4)) {
1046	case 0:
1047	div->flush_type = FLUSH_TYPE_REIMPORT;
1048	break;
1049	case 1:
1050	div->flush_type = FLUSH_TYPE_FLUSH;
1051	break;
1052	}
1053	}
1054
1055	if (div->flush_type != FLUSH_TYPE_NONE &&
1056	!(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
1057	prandom_bool(rng))
1058	div->nosimd = true;
1059
1060	switch (div->flush_type) {
1061	case FLUSH_TYPE_FLUSH:
1062	if (div->nosimd)
1063	flushtype_str = "<flush,nosimd>";
1064	else
1065	flushtype_str = "<flush>";
1066	break;
1067	case FLUSH_TYPE_REIMPORT:
1068	if (div->nosimd)
1069	flushtype_str = "<reimport,nosimd>";
1070	else
1071	flushtype_str = "<reimport>";
1072	break;
1073	default:
1074	flushtype_str = "";
1075	break;
1076	}
1077
1078	BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
1079	p += scnprintf(buf: p, size: end - p, fmt: "%s%u.%u%%@%s+%u%s", flushtype_str,
1080	this_len / 100, this_len % 100,
1081	div->offset_relative_to_alignmask ?
1082	"alignmask" : "",
1083	div->offset, this_len == remaining ? "" : ", ");
1084	remaining -= this_len;
1085	div++;
1086	} while (remaining);
1087
1088	return p;
1089	}
1090
1091	/* Generate a random testvec_config for fuzz testing */
1092	static void generate_random_testvec_config(struct rnd_state *rng,
1093	struct testvec_config *cfg,
1094	char *name, size_t max_namelen)
1095	{
1096	char *p = name;
1097	char * const end = name + max_namelen;
1098
1099	memset(cfg, 0, sizeof(*cfg));
1100
1101	cfg->name = name;
1102
1103	p += scnprintf(buf: p, size: end - p, fmt: "random:");
1104
1105	switch (prandom_u32_below(rng, ceil: 4)) {
1106	case 0:
1107	case 1:
1108	cfg->inplace_mode = OUT_OF_PLACE;
1109	break;
1110	case 2:
1111	cfg->inplace_mode = INPLACE_ONE_SGLIST;
1112	p += scnprintf(buf: p, size: end - p, fmt: " inplace_one_sglist");
1113	break;
1114	default:
1115	cfg->inplace_mode = INPLACE_TWO_SGLISTS;
1116	p += scnprintf(buf: p, size: end - p, fmt: " inplace_two_sglists");
1117	break;
1118	}
1119
1120	if (prandom_bool(rng)) {
1121	cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
1122	p += scnprintf(buf: p, size: end - p, fmt: " may_sleep");
1123	}
1124
1125	switch (prandom_u32_below(rng, ceil: 4)) {
1126	case 0:
1127	cfg->finalization_type = FINALIZATION_TYPE_FINAL;
1128	p += scnprintf(buf: p, size: end - p, fmt: " use_final");
1129	break;
1130	case 1:
1131	cfg->finalization_type = FINALIZATION_TYPE_FINUP;
1132	p += scnprintf(buf: p, size: end - p, fmt: " use_finup");
1133	break;
1134	default:
1135	cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
1136	p += scnprintf(buf: p, size: end - p, fmt: " use_digest");
1137	break;
1138	}
1139
1140	if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) {
1141	if (prandom_bool(rng)) {
1142	cfg->nosimd = true;
1143	p += scnprintf(buf: p, size: end - p, fmt: " nosimd");
1144	}
1145	if (prandom_bool(rng)) {
1146	cfg->nosimd_setkey = true;
1147	p += scnprintf(buf: p, size: end - p, fmt: " nosimd_setkey");
1148	}
1149	}
1150
1151	p += scnprintf(buf: p, size: end - p, fmt: " src_divs=[");
1152	p = generate_random_sgl_divisions(rng, divs: cfg->src_divs,
1153	ARRAY_SIZE(cfg->src_divs), p, end,
1154	gen_flushes: (cfg->finalization_type !=
1155	FINALIZATION_TYPE_DIGEST),
1156	req_flags: cfg->req_flags);
1157	p += scnprintf(buf: p, size: end - p, fmt: "]");
1158
1159	if (cfg->inplace_mode == OUT_OF_PLACE && prandom_bool(rng)) {
1160	p += scnprintf(buf: p, size: end - p, fmt: " dst_divs=[");
1161	p = generate_random_sgl_divisions(rng, divs: cfg->dst_divs,
1162	ARRAY_SIZE(cfg->dst_divs),
1163	p, end, gen_flushes: false,
1164	req_flags: cfg->req_flags);
1165	p += scnprintf(buf: p, size: end - p, fmt: "]");
1166	}
1167
1168	if (prandom_bool(rng)) {
1169	cfg->iv_offset = prandom_u32_inclusive(rng, floor: 1,
1170	MAX_ALGAPI_ALIGNMASK);
1171	p += scnprintf(buf: p, size: end - p, fmt: " iv_offset=%u", cfg->iv_offset);
1172	}
1173
1174	if (prandom_bool(rng)) {
1175	cfg->key_offset = prandom_u32_inclusive(rng, floor: 1,
1176	MAX_ALGAPI_ALIGNMASK);
1177	p += scnprintf(buf: p, size: end - p, fmt: " key_offset=%u", cfg->key_offset);
1178	}
1179
1180	WARN_ON_ONCE(!valid_testvec_config(cfg));
1181	}
1182
1183	static void crypto_disable_simd_for_test(void)
1184	{
1185	#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
1186	migrate_disable();
1187	__this_cpu_write(crypto_simd_disabled_for_test, true);
1188	#endif
1189	}
1190
1191	static void crypto_reenable_simd_for_test(void)
1192	{
1193	#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
1194	__this_cpu_write(crypto_simd_disabled_for_test, false);
1195	migrate_enable();
1196	#endif
1197	}
1198
1199	/*
1200	* Given an algorithm name, build the name of the generic implementation of that
1201	* algorithm, assuming the usual naming convention. Specifically, this appends
1202	* "-generic" to every part of the name that is not a template name. Examples:
1203	*
1204	* aes => aes-generic
1205	* cbc(aes) => cbc(aes-generic)
1206	* cts(cbc(aes)) => cts(cbc(aes-generic))
1207	* rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
1208	*
1209	* Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
1210	*/
1211	static int build_generic_driver_name(const char *algname,
1212	char driver_name[CRYPTO_MAX_ALG_NAME])
1213	{
1214	const char *in = algname;
1215	char *out = driver_name;
1216	size_t len = strlen(algname);
1217
1218	if (len >= CRYPTO_MAX_ALG_NAME)
1219	goto too_long;
1220	do {
1221	const char *in_saved = in;
1222
1223	while (*in && *in != '(' && *in != ')' && *in != ',')
1224	*out++ = *in++;
1225	if (*in != '(' && in > in_saved) {
1226	len += 8;
1227	if (len >= CRYPTO_MAX_ALG_NAME)
1228	goto too_long;
1229	memcpy(out, "-generic", 8);
1230	out += 8;
1231	}
1232	} while ((*out++ = *in++) != '\0');
1233	return 0;
1234
1235	too_long:
1236	pr_err("alg: generic driver name for \"%s\" would be too long\n",
1237	algname);
1238	return -ENAMETOOLONG;
1239	}
1240
1241	static int build_hash_sglist(struct test_sglist *tsgl,
1242	const struct hash_testvec *vec,
1243	const struct testvec_config *cfg,
1244	unsigned int alignmask,
1245	const struct test_sg_division *divs[XBUFSIZE])
1246	{
1247	struct kvec kv;
1248	struct iov_iter input;
1249
1250	kv.iov_base = (void *)vec->plaintext;
1251	kv.iov_len = vec->psize;
1252	iov_iter_kvec(i: &input, ITER_SOURCE, kvec: &kv, nr_segs: 1, count: vec->psize);
1253	return build_test_sglist(tsgl, divs: cfg->src_divs, alignmask, total_len: vec->psize,
1254	data: &input, out_divs: divs);
1255	}
1256
1257	static int check_hash_result(const char *type,
1258	const u8 *result, unsigned int digestsize,
1259	const struct hash_testvec *vec,
1260	const char *vec_name,
1261	const char *driver,
1262	const struct testvec_config *cfg)
1263	{
1264	if (memcmp(p: result, q: vec->digest, size: digestsize) != 0) {
1265	pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1266	type, driver, vec_name, cfg->name);
1267	return -EINVAL;
1268	}
1269	if (!testmgr_is_poison(addr: &result[digestsize], TESTMGR_POISON_LEN)) {
1270	pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1271	type, driver, vec_name, cfg->name);
1272	return -EOVERFLOW;
1273	}
1274	return 0;
1275	}
1276
1277	static inline int check_shash_op(const char *op, int err,
1278	const char *driver, const char *vec_name,
1279	const struct testvec_config *cfg)
1280	{
1281	if (err)
1282	pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1283	driver, op, err, vec_name, cfg->name);
1284	return err;
1285	}
1286
1287	/* Test one hash test vector in one configuration, using the shash API */
1288	static int test_shash_vec_cfg(const struct hash_testvec *vec,
1289	const char *vec_name,
1290	const struct testvec_config *cfg,
1291	struct shash_desc *desc,
1292	struct test_sglist *tsgl,
1293	u8 *hashstate)
1294	{
1295	struct crypto_shash *tfm = desc->tfm;
1296	const unsigned int digestsize = crypto_shash_digestsize(tfm);
1297	const unsigned int statesize = crypto_shash_statesize(tfm);
1298	const char *driver = crypto_shash_driver_name(tfm);
1299	const struct test_sg_division *divs[XBUFSIZE];
1300	unsigned int i;
1301	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1302	int err;
1303
1304	/* Set the key, if specified */
1305	if (vec->ksize) {
1306	err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
1307	cfg, 0);
1308	if (err) {
1309	if (err == vec->setkey_error)
1310	return 0;
1311	pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1312	driver, vec_name, vec->setkey_error, err,
1313	crypto_shash_get_flags(tfm));
1314	return err;
1315	}
1316	if (vec->setkey_error) {
1317	pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1318	driver, vec_name, vec->setkey_error);
1319	return -EINVAL;
1320	}
1321	}
1322
1323	/* Build the scatterlist for the source data */
1324	err = build_hash_sglist(tsgl, vec, cfg, alignmask: 0, divs);
1325	if (err) {
1326	pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1327	driver, vec_name, cfg->name);
1328	return err;
1329	}
1330
1331	/* Do the actual hashing */
1332
1333	testmgr_poison(addr: desc->__ctx, len: crypto_shash_descsize(tfm));
1334	testmgr_poison(addr: result, len: digestsize + TESTMGR_POISON_LEN);
1335
1336	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1337	vec->digest_error) {
1338	/* Just using digest() */
1339	if (tsgl->nents != 1)
1340	return 0;
1341	if (cfg->nosimd)
1342	crypto_disable_simd_for_test();
1343	err = crypto_shash_digest(desc, data: sg_virt(sg: &tsgl->sgl[0]),
1344	len: tsgl->sgl[0].length, out: result);
1345	if (cfg->nosimd)
1346	crypto_reenable_simd_for_test();
1347	if (err) {
1348	if (err == vec->digest_error)
1349	return 0;
1350	pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1351	driver, vec_name, vec->digest_error, err,
1352	cfg->name);
1353	return err;
1354	}
1355	if (vec->digest_error) {
1356	pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1357	driver, vec_name, vec->digest_error, cfg->name);
1358	return -EINVAL;
1359	}
1360	goto result_ready;
1361	}
1362
1363	/* Using init(), zero or more update(), then final() or finup() */
1364
1365	if (cfg->nosimd)
1366	crypto_disable_simd_for_test();
1367	err = crypto_shash_init(desc);
1368	if (cfg->nosimd)
1369	crypto_reenable_simd_for_test();
1370	err = check_shash_op(op: "init", err, driver, vec_name, cfg);
1371	if (err)
1372	return err;
1373
1374	for (i = 0; i < tsgl->nents; i++) {
1375	if (i + 1 == tsgl->nents &&
1376	cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
1377	if (divs[i]->nosimd)
1378	crypto_disable_simd_for_test();
1379	err = crypto_shash_finup(desc, data: sg_virt(sg: &tsgl->sgl[i]),
1380	len: tsgl->sgl[i].length, out: result);
1381	if (divs[i]->nosimd)
1382	crypto_reenable_simd_for_test();
1383	err = check_shash_op(op: "finup", err, driver, vec_name,
1384	cfg);
1385	if (err)
1386	return err;
1387	goto result_ready;
1388	}
1389	if (divs[i]->nosimd)
1390	crypto_disable_simd_for_test();
1391	err = crypto_shash_update(desc, data: sg_virt(sg: &tsgl->sgl[i]),
1392	len: tsgl->sgl[i].length);
1393	if (divs[i]->nosimd)
1394	crypto_reenable_simd_for_test();
1395	err = check_shash_op(op: "update", err, driver, vec_name, cfg);
1396	if (err)
1397	return err;
1398	if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1399	/* Test ->export() and ->import() */
1400	testmgr_poison(addr: hashstate + statesize,
1401	TESTMGR_POISON_LEN);
1402	err = crypto_shash_export(desc, out: hashstate);
1403	err = check_shash_op(op: "export", err, driver, vec_name,
1404	cfg);
1405	if (err)
1406	return err;
1407	if (!testmgr_is_poison(addr: hashstate + statesize,
1408	TESTMGR_POISON_LEN)) {
1409	pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1410	driver, vec_name, cfg->name);
1411	return -EOVERFLOW;
1412	}
1413	testmgr_poison(addr: desc->__ctx, len: crypto_shash_descsize(tfm));
1414	err = crypto_shash_import(desc, in: hashstate);
1415	err = check_shash_op(op: "import", err, driver, vec_name,
1416	cfg);
1417	if (err)
1418	return err;
1419	}
1420	}
1421
1422	if (cfg->nosimd)
1423	crypto_disable_simd_for_test();
1424	err = crypto_shash_final(desc, out: result);
1425	if (cfg->nosimd)
1426	crypto_reenable_simd_for_test();
1427	err = check_shash_op(op: "final", err, driver, vec_name, cfg);
1428	if (err)
1429	return err;
1430	result_ready:
1431	return check_hash_result(type: "shash", result, digestsize, vec, vec_name,
1432	driver, cfg);
1433	}
1434
1435	static int do_ahash_op(int (*op)(struct ahash_request *req),
1436	struct ahash_request *req,
1437	struct crypto_wait *wait, bool nosimd)
1438	{
1439	int err;
1440
1441	if (nosimd)
1442	crypto_disable_simd_for_test();
1443
1444	err = op(req);
1445
1446	if (nosimd)
1447	crypto_reenable_simd_for_test();
1448
1449	return crypto_wait_req(err, wait);
1450	}
1451
1452	static int check_nonfinal_ahash_op(const char *op, int err,
1453	u8 *result, unsigned int digestsize,
1454	const char *driver, const char *vec_name,
1455	const struct testvec_config *cfg)
1456	{
1457	if (err) {
1458	pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1459	driver, op, err, vec_name, cfg->name);
1460	return err;
1461	}
1462	if (!testmgr_is_poison(addr: result, len: digestsize)) {
1463	pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1464	driver, op, vec_name, cfg->name);
1465	return -EINVAL;
1466	}
1467	return 0;
1468	}
1469
1470	/* Test one hash test vector in one configuration, using the ahash API */
1471	static int test_ahash_vec_cfg(const struct hash_testvec *vec,
1472	const char *vec_name,
1473	const struct testvec_config *cfg,
1474	struct ahash_request *req,
1475	struct test_sglist *tsgl,
1476	u8 *hashstate)
1477	{
1478	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1479	const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1480	const unsigned int statesize = crypto_ahash_statesize(tfm);
1481	const char *driver = crypto_ahash_driver_name(tfm);
1482	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1483	const struct test_sg_division *divs[XBUFSIZE];
1484	DECLARE_CRYPTO_WAIT(wait);
1485	unsigned int i;
1486	struct scatterlist *pending_sgl;
1487	unsigned int pending_len;
1488	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1489	int err;
1490
1491	/* Set the key, if specified */
1492	if (vec->ksize) {
1493	err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
1494	cfg, 0);
1495	if (err) {
1496	if (err == vec->setkey_error)
1497	return 0;
1498	pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1499	driver, vec_name, vec->setkey_error, err,
1500	crypto_ahash_get_flags(tfm));
1501	return err;
1502	}
1503	if (vec->setkey_error) {
1504	pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1505	driver, vec_name, vec->setkey_error);
1506	return -EINVAL;
1507	}
1508	}
1509
1510	/* Build the scatterlist for the source data */
1511	err = build_hash_sglist(tsgl, vec, cfg, alignmask: 0, divs);
1512	if (err) {
1513	pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1514	driver, vec_name, cfg->name);
1515	return err;
1516	}
1517
1518	/* Do the actual hashing */
1519
1520	testmgr_poison(addr: req->__ctx, len: crypto_ahash_reqsize(tfm));
1521	testmgr_poison(addr: result, len: digestsize + TESTMGR_POISON_LEN);
1522
1523	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1524	vec->digest_error) {
1525	/* Just using digest() */
1526	ahash_request_set_callback(req, flags: req_flags, compl: crypto_req_done,
1527	data: &wait);
1528	ahash_request_set_crypt(req, src: tsgl->sgl, result, nbytes: vec->psize);
1529	err = do_ahash_op(op: crypto_ahash_digest, req, wait: &wait, nosimd: cfg->nosimd);
1530	if (err) {
1531	if (err == vec->digest_error)
1532	return 0;
1533	pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1534	driver, vec_name, vec->digest_error, err,
1535	cfg->name);
1536	return err;
1537	}
1538	if (vec->digest_error) {
1539	pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1540	driver, vec_name, vec->digest_error, cfg->name);
1541	return -EINVAL;
1542	}
1543	goto result_ready;
1544	}
1545
1546	/* Using init(), zero or more update(), then final() or finup() */
1547
1548	ahash_request_set_callback(req, flags: req_flags, compl: crypto_req_done, data: &wait);
1549	ahash_request_set_crypt(req, NULL, result, nbytes: 0);
1550	err = do_ahash_op(op: crypto_ahash_init, req, wait: &wait, nosimd: cfg->nosimd);
1551	err = check_nonfinal_ahash_op(op: "init", err, result, digestsize,
1552	driver, vec_name, cfg);
1553	if (err)
1554	return err;
1555
1556	pending_sgl = NULL;
1557	pending_len = 0;
1558	for (i = 0; i < tsgl->nents; i++) {
1559	if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1560	pending_sgl != NULL) {
1561	/* update() with the pending data */
1562	ahash_request_set_callback(req, flags: req_flags,
1563	compl: crypto_req_done, data: &wait);
1564	ahash_request_set_crypt(req, src: pending_sgl, result,
1565	nbytes: pending_len);
1566	err = do_ahash_op(op: crypto_ahash_update, req, wait: &wait,
1567	nosimd: divs[i]->nosimd);
1568	err = check_nonfinal_ahash_op(op: "update", err,
1569	result, digestsize,
1570	driver, vec_name, cfg);
1571	if (err)
1572	return err;
1573	pending_sgl = NULL;
1574	pending_len = 0;
1575	}
1576	if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1577	/* Test ->export() and ->import() */
1578	testmgr_poison(addr: hashstate + statesize,
1579	TESTMGR_POISON_LEN);
1580	err = crypto_ahash_export(req, out: hashstate);
1581	err = check_nonfinal_ahash_op(op: "export", err,
1582	result, digestsize,
1583	driver, vec_name, cfg);
1584	if (err)
1585	return err;
1586	if (!testmgr_is_poison(addr: hashstate + statesize,
1587	TESTMGR_POISON_LEN)) {
1588	pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1589	driver, vec_name, cfg->name);
1590	return -EOVERFLOW;
1591	}
1592
1593	testmgr_poison(addr: req->__ctx, len: crypto_ahash_reqsize(tfm));
1594	err = crypto_ahash_import(req, in: hashstate);
1595	err = check_nonfinal_ahash_op(op: "import", err,
1596	result, digestsize,
1597	driver, vec_name, cfg);
1598	if (err)
1599	return err;
1600	}
1601	if (pending_sgl == NULL)
1602	pending_sgl = &tsgl->sgl[i];
1603	pending_len += tsgl->sgl[i].length;
1604	}
1605
1606	ahash_request_set_callback(req, flags: req_flags, compl: crypto_req_done, data: &wait);
1607	ahash_request_set_crypt(req, src: pending_sgl, result, nbytes: pending_len);
1608	if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1609	/* finish with update() and final() */
1610	err = do_ahash_op(op: crypto_ahash_update, req, wait: &wait, nosimd: cfg->nosimd);
1611	err = check_nonfinal_ahash_op(op: "update", err, result, digestsize,
1612	driver, vec_name, cfg);
1613	if (err)
1614	return err;
1615	err = do_ahash_op(op: crypto_ahash_final, req, wait: &wait, nosimd: cfg->nosimd);
1616	if (err) {
1617	pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1618	driver, err, vec_name, cfg->name);
1619	return err;
1620	}
1621	} else {
1622	/* finish with finup() */
1623	err = do_ahash_op(op: crypto_ahash_finup, req, wait: &wait, nosimd: cfg->nosimd);
1624	if (err) {
1625	pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1626	driver, err, vec_name, cfg->name);
1627	return err;
1628	}
1629	}
1630
1631	result_ready:
1632	return check_hash_result(type: "ahash", result, digestsize, vec, vec_name,
1633	driver, cfg);
1634	}
1635
1636	static int test_hash_vec_cfg(const struct hash_testvec *vec,
1637	const char *vec_name,
1638	const struct testvec_config *cfg,
1639	struct ahash_request *req,
1640	struct shash_desc *desc,
1641	struct test_sglist *tsgl,
1642	u8 *hashstate)
1643	{
1644	int err;
1645
1646	/*
1647	* For algorithms implemented as "shash", most bugs will be detected by
1648	* both the shash and ahash tests. Test the shash API first so that the
1649	* failures involve less indirection, so are easier to debug.
1650	*/
1651
1652	if (desc) {
1653	err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl,
1654	hashstate);
1655	if (err)
1656	return err;
1657	}
1658
1659	return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate);
1660	}
1661
1662	static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num,
1663	struct ahash_request *req, struct shash_desc *desc,
1664	struct test_sglist *tsgl, u8 *hashstate)
1665	{
1666	char vec_name[16];
1667	unsigned int i;
1668	int err;
1669
1670	sprintf(buf: vec_name, fmt: "%u", vec_num);
1671
1672	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1673	err = test_hash_vec_cfg(vec, vec_name,
1674	cfg: &default_hash_testvec_configs[i],
1675	req, desc, tsgl, hashstate);
1676	if (err)
1677	return err;
1678	}
1679
1680	if (!noslowtests) {
1681	struct rnd_state rng;
1682	struct testvec_config cfg;
1683	char cfgname[TESTVEC_CONFIG_NAMELEN];
1684
1685	init_rnd_state(rng: &rng);
1686
1687	for (i = 0; i < fuzz_iterations; i++) {
1688	generate_random_testvec_config(rng: &rng, cfg: &cfg, name: cfgname,
1689	max_namelen: sizeof(cfgname));
1690	err = test_hash_vec_cfg(vec, vec_name, cfg: &cfg,
1691	req, desc, tsgl, hashstate);
1692	if (err)
1693	return err;
1694	cond_resched();
1695	}
1696	}
1697	return 0;
1698	}
1699
1700	/*
1701	* Generate a hash test vector from the given implementation.
1702	* Assumes the buffers in 'vec' were already allocated.
1703	*/
1704	static void generate_random_hash_testvec(struct rnd_state *rng,
1705	struct ahash_request *req,
1706	struct hash_testvec *vec,
1707	unsigned int maxkeysize,
1708	unsigned int maxdatasize,
1709	char *name, size_t max_namelen)
1710	{
1711	/* Data */
1712	vec->psize = generate_random_length(rng, max_len: maxdatasize);
1713	generate_random_bytes(rng, buf: (u8 *)vec->plaintext, count: vec->psize);
1714
1715	/*
1716	* Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1717	* If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1718	*/
1719	vec->setkey_error = 0;
1720	vec->ksize = 0;
1721	if (maxkeysize) {
1722	vec->ksize = maxkeysize;
1723	if (prandom_u32_below(rng, ceil: 4) == 0)
1724	vec->ksize = prandom_u32_inclusive(rng, floor: 1, ceil: maxkeysize);
1725	generate_random_bytes(rng, buf: (u8 *)vec->key, count: vec->ksize);
1726
1727	vec->setkey_error = crypto_ahash_setkey(
1728	tfm: crypto_ahash_reqtfm(req), key: vec->key, keylen: vec->ksize);
1729	/* If the key couldn't be set, no need to continue to digest. */
1730	if (vec->setkey_error)
1731	goto done;
1732	}
1733
1734	/* Digest */
1735	vec->digest_error = crypto_hash_digest(
1736	tfm: crypto_ahash_reqtfm(req), data: vec->plaintext,
1737	len: vec->psize, out: (u8 *)vec->digest);
1738	done:
1739	snprintf(buf: name, size: max_namelen, fmt: "\"random: psize=%u ksize=%u\"",
1740	vec->psize, vec->ksize);
1741	}
1742
1743	/*
1744	* Test the hash algorithm represented by @req against the corresponding generic
1745	* implementation, if one is available.
1746	*/
1747	static int test_hash_vs_generic_impl(const char *generic_driver,
1748	unsigned int maxkeysize,
1749	struct ahash_request *req,
1750	struct shash_desc *desc,
1751	struct test_sglist *tsgl,
1752	u8 *hashstate)
1753	{
1754	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1755	const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1756	const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1757	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1758	const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1759	const char *driver = crypto_ahash_driver_name(tfm);
1760	struct rnd_state rng;
1761	char _generic_driver[CRYPTO_MAX_ALG_NAME];
1762	struct ahash_request *generic_req = NULL;
1763	struct crypto_ahash *generic_tfm = NULL;
1764	unsigned int i;
1765	struct hash_testvec vec = { 0 };
1766	char vec_name[64];
1767	struct testvec_config *cfg;
1768	char cfgname[TESTVEC_CONFIG_NAMELEN];
1769	int err;
1770
1771	if (noslowtests)
1772	return 0;
1773
1774	init_rnd_state(rng: &rng);
1775
1776	if (!generic_driver) { /* Use default naming convention? */
1777	err = build_generic_driver_name(algname, driver_name: _generic_driver);
1778	if (err)
1779	return err;
1780	generic_driver = _generic_driver;
1781	}
1782
1783	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1784	return 0;
1785
1786	generic_tfm = crypto_alloc_ahash(alg_name: generic_driver, type: 0, mask: 0);
1787	if (IS_ERR(ptr: generic_tfm)) {
1788	err = PTR_ERR(ptr: generic_tfm);
1789	if (err == -ENOENT) {
1790	pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1791	driver, generic_driver);
1792	return 0;
1793	}
1794	pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1795	generic_driver, algname, err);
1796	return err;
1797	}
1798
1799	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1800	if (!cfg) {
1801	err = -ENOMEM;
1802	goto out;
1803	}
1804
1805	generic_req = ahash_request_alloc(generic_tfm, GFP_KERNEL);
1806	if (!generic_req) {
1807	err = -ENOMEM;
1808	goto out;
1809	}
1810
1811	/* Check the algorithm properties for consistency. */
1812
1813	if (digestsize != crypto_ahash_digestsize(tfm: generic_tfm)) {
1814	pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1815	driver, digestsize,
1816	crypto_ahash_digestsize(generic_tfm));
1817	err = -EINVAL;
1818	goto out;
1819	}
1820
1821	if (blocksize != crypto_ahash_blocksize(tfm: generic_tfm)) {
1822	pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1823	driver, blocksize, crypto_ahash_blocksize(generic_tfm));
1824	err = -EINVAL;
1825	goto out;
1826	}
1827
1828	/*
1829	* Now generate test vectors using the generic implementation, and test
1830	* the other implementation against them.
1831	*/
1832
1833	vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1834	vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1835	vec.digest = kmalloc(digestsize, GFP_KERNEL);
1836	if (!vec.key || !vec.plaintext || !vec.digest) {
1837	err = -ENOMEM;
1838	goto out;
1839	}
1840
1841	for (i = 0; i < fuzz_iterations * 8; i++) {
1842	generate_random_hash_testvec(rng: &rng, req: generic_req, vec: &vec,
1843	maxkeysize, maxdatasize,
1844	name: vec_name, max_namelen: sizeof(vec_name));
1845	generate_random_testvec_config(rng: &rng, cfg, name: cfgname,
1846	max_namelen: sizeof(cfgname));
1847
1848	err = test_hash_vec_cfg(vec: &vec, vec_name, cfg,
1849	req, desc, tsgl, hashstate);
1850	if (err)
1851	goto out;
1852	cond_resched();
1853	}
1854	err = 0;
1855	out:
1856	kfree(objp: cfg);
1857	kfree(objp: vec.key);
1858	kfree(objp: vec.plaintext);
1859	kfree(objp: vec.digest);
1860	ahash_request_free(req: generic_req);
1861	crypto_free_ahash(tfm: generic_tfm);
1862	return err;
1863	}
1864
1865	static int alloc_shash(const char *driver, u32 type, u32 mask,
1866	struct crypto_shash **tfm_ret,
1867	struct shash_desc **desc_ret)
1868	{
1869	struct crypto_shash *tfm;
1870	struct shash_desc *desc;
1871
1872	tfm = crypto_alloc_shash(alg_name: driver, type, mask);
1873	if (IS_ERR(ptr: tfm)) {
1874	if (PTR_ERR(ptr: tfm) == -ENOENT || PTR_ERR(ptr: tfm) == -EEXIST) {
1875	/*
1876	* This algorithm is only available through the ahash
1877	* API, not the shash API, so skip the shash tests.
1878	*/
1879	return 0;
1880	}
1881	pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
1882	driver, PTR_ERR(tfm));
1883	return PTR_ERR(ptr: tfm);
1884	}
1885
1886	desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
1887	if (!desc) {
1888	crypto_free_shash(tfm);
1889	return -ENOMEM;
1890	}
1891	desc->tfm = tfm;
1892
1893	*tfm_ret = tfm;
1894	*desc_ret = desc;
1895	return 0;
1896	}
1897
1898	static int __alg_test_hash(const struct hash_testvec *vecs,
1899	unsigned int num_vecs, const char *driver,
1900	u32 type, u32 mask,
1901	const char *generic_driver, unsigned int maxkeysize)
1902	{
1903	struct crypto_ahash *atfm = NULL;
1904	struct ahash_request *req = NULL;
1905	struct crypto_shash *stfm = NULL;
1906	struct shash_desc *desc = NULL;
1907	struct test_sglist *tsgl = NULL;
1908	u8 *hashstate = NULL;
1909	unsigned int statesize;
1910	unsigned int i;
1911	int err;
1912
1913	/*
1914	* Always test the ahash API. This works regardless of whether the
1915	* algorithm is implemented as ahash or shash.
1916	*/
1917
1918	atfm = crypto_alloc_ahash(alg_name: driver, type, mask);
1919	if (IS_ERR(ptr: atfm)) {
1920	if (PTR_ERR(ptr: atfm) == -ENOENT)
1921	return 0;
1922	pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1923	driver, PTR_ERR(atfm));
1924	return PTR_ERR(ptr: atfm);
1925	}
1926	driver = crypto_ahash_driver_name(tfm: atfm);
1927
1928	req = ahash_request_alloc(atfm, GFP_KERNEL);
1929	if (!req) {
1930	pr_err("alg: hash: failed to allocate request for %s\n",
1931	driver);
1932	err = -ENOMEM;
1933	goto out;
1934	}
1935
1936	/*
1937	* If available also test the shash API, to cover corner cases that may
1938	* be missed by testing the ahash API only.
1939	*/
1940	err = alloc_shash(driver, type, mask, tfm_ret: &stfm, desc_ret: &desc);
1941	if (err)
1942	goto out;
1943
1944	tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1945	if (!tsgl || init_test_sglist(tsgl) != 0) {
1946	pr_err("alg: hash: failed to allocate test buffers for %s\n",
1947	driver);
1948	kfree(objp: tsgl);
1949	tsgl = NULL;
1950	err = -ENOMEM;
1951	goto out;
1952	}
1953
1954	statesize = crypto_ahash_statesize(tfm: atfm);
1955	if (stfm)
1956	statesize = max(statesize, crypto_shash_statesize(stfm));
1957	hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
1958	if (!hashstate) {
1959	pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1960	driver);
1961	err = -ENOMEM;
1962	goto out;
1963	}
1964
1965	for (i = 0; i < num_vecs; i++) {
1966	if (fips_enabled && vecs[i].fips_skip)
1967	continue;
1968
1969	err = test_hash_vec(vec: &vecs[i], vec_num: i, req, desc, tsgl, hashstate);
1970	if (err)
1971	goto out;
1972	cond_resched();
1973	}
1974	err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req,
1975	desc, tsgl, hashstate);
1976	out:
1977	kfree(objp: hashstate);
1978	if (tsgl) {
1979	destroy_test_sglist(tsgl);
1980	kfree(objp: tsgl);
1981	}
1982	kfree(objp: desc);
1983	crypto_free_shash(tfm: stfm);
1984	ahash_request_free(req);
1985	crypto_free_ahash(tfm: atfm);
1986	return err;
1987	}
1988
1989	static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1990	u32 type, u32 mask)
1991	{
1992	const struct hash_testvec *template = desc->suite.hash.vecs;
1993	unsigned int tcount = desc->suite.hash.count;
1994	unsigned int nr_unkeyed, nr_keyed;
1995	unsigned int maxkeysize = 0;
1996	int err;
1997
1998	/*
1999	* For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
2000	* first, before setting a key on the tfm. To make this easier, we
2001	* require that the unkeyed test vectors (if any) are listed first.
2002	*/
2003
2004	for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
2005	if (template[nr_unkeyed].ksize)
2006	break;
2007	}
2008	for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
2009	if (!template[nr_unkeyed + nr_keyed].ksize) {
2010	pr_err("alg: hash: test vectors for %s out of order, "
2011	"unkeyed ones must come first\n", desc->alg);
2012	return -EINVAL;
2013	}
2014	maxkeysize = max_t(unsigned int, maxkeysize,
2015	template[nr_unkeyed + nr_keyed].ksize);
2016	}
2017
2018	err = 0;
2019	if (nr_unkeyed) {
2020	err = __alg_test_hash(vecs: template, num_vecs: nr_unkeyed, driver, type, mask,
2021	generic_driver: desc->generic_driver, maxkeysize);
2022	template += nr_unkeyed;
2023	}
2024
2025	if (!err && nr_keyed)
2026	err = __alg_test_hash(vecs: template, num_vecs: nr_keyed, driver, type, mask,
2027	generic_driver: desc->generic_driver, maxkeysize);
2028
2029	return err;
2030	}
2031
2032	static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec,
2033	const char *vec_name,
2034	const struct testvec_config *cfg,
2035	struct aead_request *req,
2036	struct cipher_test_sglists *tsgls)
2037	{
2038	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2039	const unsigned int alignmask = crypto_aead_alignmask(tfm);
2040	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2041	const unsigned int authsize = vec->clen - vec->plen;
2042	const char *driver = crypto_aead_driver_name(tfm);
2043	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2044	const char *op = enc ? "encryption" : "decryption";
2045	DECLARE_CRYPTO_WAIT(wait);
2046	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2047	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2048	cfg->iv_offset +
2049	(cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2050	struct kvec input[2];
2051	int err;
2052
2053	/* Set the key */
2054	if (vec->wk)
2055	crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2056	else
2057	crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2058
2059	err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
2060	cfg, alignmask);
2061	if (err && err != vec->setkey_error) {
2062	pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2063	driver, vec_name, vec->setkey_error, err,
2064	crypto_aead_get_flags(tfm));
2065	return err;
2066	}
2067	if (!err && vec->setkey_error) {
2068	pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2069	driver, vec_name, vec->setkey_error);
2070	return -EINVAL;
2071	}
2072
2073	/* Set the authentication tag size */
2074	err = crypto_aead_setauthsize(tfm, authsize);
2075	if (err && err != vec->setauthsize_error) {
2076	pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
2077	driver, vec_name, vec->setauthsize_error, err);
2078	return err;
2079	}
2080	if (!err && vec->setauthsize_error) {
2081	pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
2082	driver, vec_name, vec->setauthsize_error);
2083	return -EINVAL;
2084	}
2085
2086	if (vec->setkey_error || vec->setauthsize_error)
2087	return 0;
2088
2089	/* The IV must be copied to a buffer, as the algorithm may modify it */
2090	if (WARN_ON(ivsize > MAX_IVLEN))
2091	return -EINVAL;
2092	if (vec->iv)
2093	memcpy(iv, vec->iv, ivsize);
2094	else
2095	memset(iv, 0, ivsize);
2096
2097	/* Build the src/dst scatterlists */
2098	input[0].iov_base = (void *)vec->assoc;
2099	input[0].iov_len = vec->alen;
2100	input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2101	input[1].iov_len = enc ? vec->plen : vec->clen;
2102	err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2103	src_total_len: vec->alen + (enc ? vec->plen :
2104	vec->clen),
2105	dst_total_len: vec->alen + (enc ? vec->clen :
2106	vec->plen),
2107	inputs: input, nr_inputs: 2);
2108	if (err) {
2109	pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2110	driver, op, vec_name, cfg->name);
2111	return err;
2112	}
2113
2114	/* Do the actual encryption or decryption */
2115	testmgr_poison(addr: req->__ctx, len: crypto_aead_reqsize(tfm));
2116	aead_request_set_callback(req, flags: req_flags, compl: crypto_req_done, data: &wait);
2117	aead_request_set_crypt(req, src: tsgls->src.sgl_ptr, dst: tsgls->dst.sgl_ptr,
2118	cryptlen: enc ? vec->plen : vec->clen, iv);
2119	aead_request_set_ad(req, assoclen: vec->alen);
2120	if (cfg->nosimd)
2121	crypto_disable_simd_for_test();
2122	err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
2123	if (cfg->nosimd)
2124	crypto_reenable_simd_for_test();
2125	err = crypto_wait_req(err, wait: &wait);
2126
2127	/* Check that the algorithm didn't overwrite things it shouldn't have */
2128	if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
2129	req->assoclen != vec->alen ||
2130	req->iv != iv ||
2131	req->src != tsgls->src.sgl_ptr ||
2132	req->dst != tsgls->dst.sgl_ptr ||
2133	crypto_aead_reqtfm(req) != tfm ||
2134	req->base.complete != crypto_req_done ||
2135	req->base.flags != req_flags ||
2136	req->base.data != &wait) {
2137	pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2138	driver, op, vec_name, cfg->name);
2139	if (req->cryptlen != (enc ? vec->plen : vec->clen))
2140	pr_err("alg: aead: changed 'req->cryptlen'\n");
2141	if (req->assoclen != vec->alen)
2142	pr_err("alg: aead: changed 'req->assoclen'\n");
2143	if (req->iv != iv)
2144	pr_err("alg: aead: changed 'req->iv'\n");
2145	if (req->src != tsgls->src.sgl_ptr)
2146	pr_err("alg: aead: changed 'req->src'\n");
2147	if (req->dst != tsgls->dst.sgl_ptr)
2148	pr_err("alg: aead: changed 'req->dst'\n");
2149	if (crypto_aead_reqtfm(req) != tfm)
2150	pr_err("alg: aead: changed 'req->base.tfm'\n");
2151	if (req->base.complete != crypto_req_done)
2152	pr_err("alg: aead: changed 'req->base.complete'\n");
2153	if (req->base.flags != req_flags)
2154	pr_err("alg: aead: changed 'req->base.flags'\n");
2155	if (req->base.data != &wait)
2156	pr_err("alg: aead: changed 'req->base.data'\n");
2157	return -EINVAL;
2158	}
2159	if (is_test_sglist_corrupted(tsgl: &tsgls->src)) {
2160	pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2161	driver, op, vec_name, cfg->name);
2162	return -EINVAL;
2163	}
2164	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2165	is_test_sglist_corrupted(tsgl: &tsgls->dst)) {
2166	pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2167	driver, op, vec_name, cfg->name);
2168	return -EINVAL;
2169	}
2170
2171	/* Check for unexpected success or failure, or wrong error code */
2172	if ((err == 0 && vec->novrfy) ||
2173	(err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
2174	char expected_error[32];
2175
2176	if (vec->novrfy &&
2177	vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
2178	sprintf(buf: expected_error, fmt: "-EBADMSG or %d",
2179	vec->crypt_error);
2180	else if (vec->novrfy)
2181	sprintf(buf: expected_error, fmt: "-EBADMSG");
2182	else
2183	sprintf(buf: expected_error, fmt: "%d", vec->crypt_error);
2184	if (err) {
2185	pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
2186	driver, op, vec_name, expected_error, err,
2187	cfg->name);
2188	return err;
2189	}
2190	pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
2191	driver, op, vec_name, expected_error, cfg->name);
2192	return -EINVAL;
2193	}
2194	if (err) /* Expectedly failed. */
2195	return 0;
2196
2197	/* Check for the correct output (ciphertext or plaintext) */
2198	err = verify_correct_output(tsgl: &tsgls->dst, expected_output: enc ? vec->ctext : vec->ptext,
2199	len_to_check: enc ? vec->clen : vec->plen,
2200	unchecked_prefix_len: vec->alen,
2201	check_poison: enc || cfg->inplace_mode == OUT_OF_PLACE);
2202	if (err == -EOVERFLOW) {
2203	pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2204	driver, op, vec_name, cfg->name);
2205	return err;
2206	}
2207	if (err) {
2208	pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2209	driver, op, vec_name, cfg->name);
2210	return err;
2211	}
2212
2213	return 0;
2214	}
2215
2216	static int test_aead_vec(int enc, const struct aead_testvec *vec,
2217	unsigned int vec_num, struct aead_request *req,
2218	struct cipher_test_sglists *tsgls)
2219	{
2220	char vec_name[16];
2221	unsigned int i;
2222	int err;
2223
2224	if (enc && vec->novrfy)
2225	return 0;
2226
2227	sprintf(buf: vec_name, fmt: "%u", vec_num);
2228
2229	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2230	err = test_aead_vec_cfg(enc, vec, vec_name,
2231	cfg: &default_cipher_testvec_configs[i],
2232	req, tsgls);
2233	if (err)
2234	return err;
2235	}
2236
2237	if (!noslowtests) {
2238	struct rnd_state rng;
2239	struct testvec_config cfg;
2240	char cfgname[TESTVEC_CONFIG_NAMELEN];
2241
2242	init_rnd_state(rng: &rng);
2243
2244	for (i = 0; i < fuzz_iterations; i++) {
2245	generate_random_testvec_config(rng: &rng, cfg: &cfg, name: cfgname,
2246	max_namelen: sizeof(cfgname));
2247	err = test_aead_vec_cfg(enc, vec, vec_name,
2248	cfg: &cfg, req, tsgls);
2249	if (err)
2250	return err;
2251	cond_resched();
2252	}
2253	}
2254	return 0;
2255	}
2256
2257	struct aead_slow_tests_ctx {
2258	struct rnd_state rng;
2259	struct aead_request *req;
2260	struct crypto_aead *tfm;
2261	const struct alg_test_desc *test_desc;
2262	struct cipher_test_sglists *tsgls;
2263	unsigned int maxdatasize;
2264	unsigned int maxkeysize;
2265
2266	struct aead_testvec vec;
2267	char vec_name[64];
2268	char cfgname[TESTVEC_CONFIG_NAMELEN];
2269	struct testvec_config cfg;
2270	};
2271
2272	/*
2273	* Make at least one random change to a (ciphertext, AAD) pair. "Ciphertext"
2274	* here means the full ciphertext including the authentication tag. The
2275	* authentication tag (and hence also the ciphertext) is assumed to be nonempty.
2276	*/
2277	static void mutate_aead_message(struct rnd_state *rng,
2278	struct aead_testvec *vec, bool aad_iv,
2279	unsigned int ivsize)
2280	{
2281	const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
2282	const unsigned int authsize = vec->clen - vec->plen;
2283
2284	if (prandom_bool(rng) && vec->alen > aad_tail_size) {
2285	/* Mutate the AAD */
2286	flip_random_bit(rng, buf: (u8 *)vec->assoc,
2287	size: vec->alen - aad_tail_size);
2288	if (prandom_bool(rng))
2289	return;
2290	}
2291	if (prandom_bool(rng)) {
2292	/* Mutate auth tag (assuming it's at the end of ciphertext) */
2293	flip_random_bit(rng, buf: (u8 *)vec->ctext + vec->plen, size: authsize);
2294	} else {
2295	/* Mutate any part of the ciphertext */
2296	flip_random_bit(rng, buf: (u8 *)vec->ctext, size: vec->clen);
2297	}
2298	}
2299
2300	/*
2301	* Minimum authentication tag size in bytes at which we assume that we can
2302	* reliably generate inauthentic messages, i.e. not generate an authentic
2303	* message by chance.
2304	*/
2305	#define MIN_COLLISION_FREE_AUTHSIZE 8
2306
2307	static void generate_aead_message(struct rnd_state *rng,
2308	struct aead_request *req,
2309	const struct aead_test_suite *suite,
2310	struct aead_testvec *vec,
2311	bool prefer_inauthentic)
2312	{
2313	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2314	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2315	const unsigned int authsize = vec->clen - vec->plen;
2316	const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
2317	(prefer_inauthentic ||
2318	prandom_u32_below(rng, ceil: 4) == 0);
2319
2320	/* Generate the AAD. */
2321	generate_random_bytes(rng, buf: (u8 *)vec->assoc, count: vec->alen);
2322	if (suite->aad_iv && vec->alen >= ivsize)
2323	/* Avoid implementation-defined behavior. */
2324	memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
2325
2326	if (inauthentic && prandom_bool(rng)) {
2327	/* Generate a random ciphertext. */
2328	generate_random_bytes(rng, buf: (u8 *)vec->ctext, count: vec->clen);
2329	} else {
2330	int i = 0;
2331	struct scatterlist src[2], dst;
2332	u8 iv[MAX_IVLEN];
2333	DECLARE_CRYPTO_WAIT(wait);
2334
2335	/* Generate a random plaintext and encrypt it. */
2336	sg_init_table(src, 2);
2337	if (vec->alen)
2338	sg_set_buf(sg: &src[i++], buf: vec->assoc, buflen: vec->alen);
2339	if (vec->plen) {
2340	generate_random_bytes(rng, buf: (u8 *)vec->ptext, count: vec->plen);
2341	sg_set_buf(sg: &src[i++], buf: vec->ptext, buflen: vec->plen);
2342	}
2343	sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
2344	memcpy(iv, vec->iv, ivsize);
2345	aead_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
2346	aead_request_set_crypt(req, src, dst: &dst, cryptlen: vec->plen, iv);
2347	aead_request_set_ad(req, assoclen: vec->alen);
2348	vec->crypt_error = crypto_wait_req(err: crypto_aead_encrypt(req),
2349	wait: &wait);
2350	/* If encryption failed, we're done. */
2351	if (vec->crypt_error != 0)
2352	return;
2353	memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
2354	if (!inauthentic)
2355	return;
2356	/*
2357	* Mutate the authentic (ciphertext, AAD) pair to get an
2358	* inauthentic one.
2359	*/
2360	mutate_aead_message(rng, vec, aad_iv: suite->aad_iv, ivsize);
2361	}
2362	vec->novrfy = 1;
2363	if (suite->einval_allowed)
2364	vec->crypt_error = -EINVAL;
2365	}
2366
2367	/*
2368	* Generate an AEAD test vector 'vec' using the implementation specified by
2369	* 'req'. The buffers in 'vec' must already be allocated.
2370	*
2371	* If 'prefer_inauthentic' is true, then this function will generate inauthentic
2372	* test vectors (i.e. vectors with 'vec->novrfy=1') more often.
2373	*/
2374	static void generate_random_aead_testvec(struct rnd_state *rng,
2375	struct aead_request *req,
2376	struct aead_testvec *vec,
2377	const struct aead_test_suite *suite,
2378	unsigned int maxkeysize,
2379	unsigned int maxdatasize,
2380	char *name, size_t max_namelen,
2381	bool prefer_inauthentic)
2382	{
2383	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2384	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2385	const unsigned int maxauthsize = crypto_aead_maxauthsize(aead: tfm);
2386	unsigned int authsize;
2387	unsigned int total_len;
2388
2389	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2390	vec->klen = maxkeysize;
2391	if (prandom_u32_below(rng, ceil: 4) == 0)
2392	vec->klen = prandom_u32_below(rng, ceil: maxkeysize + 1);
2393	generate_random_bytes(rng, buf: (u8 *)vec->key, count: vec->klen);
2394	vec->setkey_error = crypto_aead_setkey(tfm, key: vec->key, keylen: vec->klen);
2395
2396	/* IV */
2397	generate_random_bytes(rng, buf: (u8 *)vec->iv, count: ivsize);
2398
2399	/* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
2400	authsize = maxauthsize;
2401	if (prandom_u32_below(rng, ceil: 4) == 0)
2402	authsize = prandom_u32_below(rng, ceil: maxauthsize + 1);
2403	if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
2404	authsize = MIN_COLLISION_FREE_AUTHSIZE;
2405	if (WARN_ON(authsize > maxdatasize))
2406	authsize = maxdatasize;
2407	maxdatasize -= authsize;
2408	vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
2409
2410	/* AAD, plaintext, and ciphertext lengths */
2411	total_len = generate_random_length(rng, max_len: maxdatasize);
2412	if (prandom_u32_below(rng, ceil: 4) == 0)
2413	vec->alen = 0;
2414	else
2415	vec->alen = generate_random_length(rng, max_len: total_len);
2416	vec->plen = total_len - vec->alen;
2417	vec->clen = vec->plen + authsize;
2418
2419	/*
2420	* Generate the AAD, plaintext, and ciphertext. Not applicable if the
2421	* key or the authentication tag size couldn't be set.
2422	*/
2423	vec->novrfy = 0;
2424	vec->crypt_error = 0;
2425	if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
2426	generate_aead_message(rng, req, suite, vec, prefer_inauthentic);
2427	snprintf(buf: name, size: max_namelen,
2428	fmt: "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
2429	vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
2430	}
2431
2432	static void try_to_generate_inauthentic_testvec(struct aead_slow_tests_ctx *ctx)
2433	{
2434	int i;
2435
2436	for (i = 0; i < 10; i++) {
2437	generate_random_aead_testvec(rng: &ctx->rng, req: ctx->req, vec: &ctx->vec,
2438	suite: &ctx->test_desc->suite.aead,
2439	maxkeysize: ctx->maxkeysize, maxdatasize: ctx->maxdatasize,
2440	name: ctx->vec_name,
2441	max_namelen: sizeof(ctx->vec_name), prefer_inauthentic: true);
2442	if (ctx->vec.novrfy)
2443	return;
2444	}
2445	}
2446
2447	/*
2448	* Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
2449	* result of an encryption with the key) and verify that decryption fails.
2450	*/
2451	static int test_aead_inauthentic_inputs(struct aead_slow_tests_ctx *ctx)
2452	{
2453	unsigned int i;
2454	int err;
2455
2456	for (i = 0; i < fuzz_iterations * 8; i++) {
2457	/*
2458	* Since this part of the tests isn't comparing the
2459	* implementation to another, there's no point in testing any
2460	* test vectors other than inauthentic ones (vec.novrfy=1) here.
2461	*
2462	* If we're having trouble generating such a test vector, e.g.
2463	* if the algorithm keeps rejecting the generated keys, don't
2464	* retry forever; just continue on.
2465	*/
2466	try_to_generate_inauthentic_testvec(ctx);
2467	if (ctx->vec.novrfy) {
2468	generate_random_testvec_config(rng: &ctx->rng, cfg: &ctx->cfg,
2469	name: ctx->cfgname,
2470	max_namelen: sizeof(ctx->cfgname));
2471	err = test_aead_vec_cfg(DECRYPT, vec: &ctx->vec,
2472	vec_name: ctx->vec_name, cfg: &ctx->cfg,
2473	req: ctx->req, tsgls: ctx->tsgls);
2474	if (err)
2475	return err;
2476	}
2477	cond_resched();
2478	}
2479	return 0;
2480	}
2481
2482	/*
2483	* Test the AEAD algorithm against the corresponding generic implementation, if
2484	* one is available.
2485	*/
2486	static int test_aead_vs_generic_impl(struct aead_slow_tests_ctx *ctx)
2487	{
2488	struct crypto_aead *tfm = ctx->tfm;
2489	const char *algname = crypto_aead_alg(tfm)->base.cra_name;
2490	const char *driver = crypto_aead_driver_name(tfm);
2491	const char *generic_driver = ctx->test_desc->generic_driver;
2492	char _generic_driver[CRYPTO_MAX_ALG_NAME];
2493	struct crypto_aead *generic_tfm = NULL;
2494	struct aead_request *generic_req = NULL;
2495	unsigned int i;
2496	int err;
2497
2498	if (!generic_driver) { /* Use default naming convention? */
2499	err = build_generic_driver_name(algname, driver_name: _generic_driver);
2500	if (err)
2501	return err;
2502	generic_driver = _generic_driver;
2503	}
2504
2505	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2506	return 0;
2507
2508	generic_tfm = crypto_alloc_aead(alg_name: generic_driver, type: 0, mask: 0);
2509	if (IS_ERR(ptr: generic_tfm)) {
2510	err = PTR_ERR(ptr: generic_tfm);
2511	if (err == -ENOENT) {
2512	pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
2513	driver, generic_driver);
2514	return 0;
2515	}
2516	pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
2517	generic_driver, algname, err);
2518	return err;
2519	}
2520
2521	generic_req = aead_request_alloc(tfm: generic_tfm, GFP_KERNEL);
2522	if (!generic_req) {
2523	err = -ENOMEM;
2524	goto out;
2525	}
2526
2527	/* Check the algorithm properties for consistency. */
2528
2529	if (crypto_aead_maxauthsize(aead: tfm) !=
2530	crypto_aead_maxauthsize(aead: generic_tfm)) {
2531	pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
2532	driver, crypto_aead_maxauthsize(tfm),
2533	crypto_aead_maxauthsize(generic_tfm));
2534	err = -EINVAL;
2535	goto out;
2536	}
2537
2538	if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(tfm: generic_tfm)) {
2539	pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2540	driver, crypto_aead_ivsize(tfm),
2541	crypto_aead_ivsize(generic_tfm));
2542	err = -EINVAL;
2543	goto out;
2544	}
2545
2546	if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(tfm: generic_tfm)) {
2547	pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2548	driver, crypto_aead_blocksize(tfm),
2549	crypto_aead_blocksize(generic_tfm));
2550	err = -EINVAL;
2551	goto out;
2552	}
2553
2554	/*
2555	* Now generate test vectors using the generic implementation, and test
2556	* the other implementation against them.
2557	*/
2558	for (i = 0; i < fuzz_iterations * 8; i++) {
2559	generate_random_aead_testvec(rng: &ctx->rng, req: generic_req, vec: &ctx->vec,
2560	suite: &ctx->test_desc->suite.aead,
2561	maxkeysize: ctx->maxkeysize, maxdatasize: ctx->maxdatasize,
2562	name: ctx->vec_name,
2563	max_namelen: sizeof(ctx->vec_name), prefer_inauthentic: false);
2564	generate_random_testvec_config(rng: &ctx->rng, cfg: &ctx->cfg,
2565	name: ctx->cfgname,
2566	max_namelen: sizeof(ctx->cfgname));
2567	if (!ctx->vec.novrfy) {
2568	err = test_aead_vec_cfg(ENCRYPT, vec: &ctx->vec,
2569	vec_name: ctx->vec_name, cfg: &ctx->cfg,
2570	req: ctx->req, tsgls: ctx->tsgls);
2571	if (err)
2572	goto out;
2573	}
2574	if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {
2575	err = test_aead_vec_cfg(DECRYPT, vec: &ctx->vec,
2576	vec_name: ctx->vec_name, cfg: &ctx->cfg,
2577	req: ctx->req, tsgls: ctx->tsgls);
2578	if (err)
2579	goto out;
2580	}
2581	cond_resched();
2582	}
2583	err = 0;
2584	out:
2585	crypto_free_aead(tfm: generic_tfm);
2586	aead_request_free(req: generic_req);
2587	return err;
2588	}
2589
2590	static int test_aead_slow(const struct alg_test_desc *test_desc,
2591	struct aead_request *req,
2592	struct cipher_test_sglists *tsgls)
2593	{
2594	struct aead_slow_tests_ctx *ctx;
2595	unsigned int i;
2596	int err;
2597
2598	if (noslowtests)
2599	return 0;
2600
2601	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2602	if (!ctx)
2603	return -ENOMEM;
2604	init_rnd_state(rng: &ctx->rng);
2605	ctx->req = req;
2606	ctx->tfm = crypto_aead_reqtfm(req);
2607	ctx->test_desc = test_desc;
2608	ctx->tsgls = tsgls;
2609	ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2610	ctx->maxkeysize = 0;
2611	for (i = 0; i < test_desc->suite.aead.count; i++)
2612	ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
2613	test_desc->suite.aead.vecs[i].klen);
2614
2615	ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
2616	ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
2617	ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2618	ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2619	ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2620	if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||
2621	!ctx->vec.ptext || !ctx->vec.ctext) {
2622	err = -ENOMEM;
2623	goto out;
2624	}
2625
2626	err = test_aead_vs_generic_impl(ctx);
2627	if (err)
2628	goto out;
2629
2630	err = test_aead_inauthentic_inputs(ctx);
2631	out:
2632	kfree(objp: ctx->vec.key);
2633	kfree(objp: ctx->vec.iv);
2634	kfree(objp: ctx->vec.assoc);
2635	kfree(objp: ctx->vec.ptext);
2636	kfree(objp: ctx->vec.ctext);
2637	kfree(objp: ctx);
2638	return err;
2639	}
2640
2641	static int test_aead(int enc, const struct aead_test_suite *suite,
2642	struct aead_request *req,
2643	struct cipher_test_sglists *tsgls)
2644	{
2645	unsigned int i;
2646	int err;
2647
2648	for (i = 0; i < suite->count; i++) {
2649	err = test_aead_vec(enc, vec: &suite->vecs[i], vec_num: i, req, tsgls);
2650	if (err)
2651	return err;
2652	cond_resched();
2653	}
2654	return 0;
2655	}
2656
2657	static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2658	u32 type, u32 mask)
2659	{
2660	const struct aead_test_suite *suite = &desc->suite.aead;
2661	struct crypto_aead *tfm;
2662	struct aead_request *req = NULL;
2663	struct cipher_test_sglists *tsgls = NULL;
2664	int err;
2665
2666	if (suite->count <= 0) {
2667	pr_err("alg: aead: empty test suite for %s\n", driver);
2668	return -EINVAL;
2669	}
2670
2671	tfm = crypto_alloc_aead(alg_name: driver, type, mask);
2672	if (IS_ERR(ptr: tfm)) {
2673	if (PTR_ERR(ptr: tfm) == -ENOENT)
2674	return 0;
2675	pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2676	driver, PTR_ERR(tfm));
2677	return PTR_ERR(ptr: tfm);
2678	}
2679	driver = crypto_aead_driver_name(tfm);
2680
2681	req = aead_request_alloc(tfm, GFP_KERNEL);
2682	if (!req) {
2683	pr_err("alg: aead: failed to allocate request for %s\n",
2684	driver);
2685	err = -ENOMEM;
2686	goto out;
2687	}
2688
2689	tsgls = alloc_cipher_test_sglists();
2690	if (!tsgls) {
2691	pr_err("alg: aead: failed to allocate test buffers for %s\n",
2692	driver);
2693	err = -ENOMEM;
2694	goto out;
2695	}
2696
2697	err = test_aead(ENCRYPT, suite, req, tsgls);
2698	if (err)
2699	goto out;
2700
2701	err = test_aead(DECRYPT, suite, req, tsgls);
2702	if (err)
2703	goto out;
2704
2705	err = test_aead_slow(test_desc: desc, req, tsgls);
2706	out:
2707	free_cipher_test_sglists(tsgls);
2708	aead_request_free(req);
2709	crypto_free_aead(tfm);
2710	return err;
2711	}
2712
2713	static int test_cipher(struct crypto_cipher *tfm, int enc,
2714	const struct cipher_testvec *template,
2715	unsigned int tcount)
2716	{
2717	const char *algo = crypto_tfm_alg_driver_name(tfm: crypto_cipher_tfm(tfm));
2718	unsigned int i, j, k;
2719	char *q;
2720	const char *e;
2721	const char *input, *result;
2722	void *data;
2723	char *xbuf[XBUFSIZE];
2724	int ret = -ENOMEM;
2725
2726	if (testmgr_alloc_buf(buf: xbuf))
2727	goto out_nobuf;
2728
2729	if (enc == ENCRYPT)
2730	e = "encryption";
2731	else
2732	e = "decryption";
2733
2734	j = 0;
2735	for (i = 0; i < tcount; i++) {
2736
2737	if (fips_enabled && template[i].fips_skip)
2738	continue;
2739
2740	input = enc ? template[i].ptext : template[i].ctext;
2741	result = enc ? template[i].ctext : template[i].ptext;
2742	j++;
2743
2744	ret = -EINVAL;
2745	if (WARN_ON(template[i].len > PAGE_SIZE))
2746	goto out;
2747
2748	data = xbuf[0];
2749	memcpy(data, input, template[i].len);
2750
2751	crypto_cipher_clear_flags(tfm, flags: ~0);
2752	if (template[i].wk)
2753	crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2754
2755	ret = crypto_cipher_setkey(tfm, key: template[i].key,
2756	keylen: template[i].klen);
2757	if (ret) {
2758	if (ret == template[i].setkey_error)
2759	continue;
2760	pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2761	algo, j, template[i].setkey_error, ret,
2762	crypto_cipher_get_flags(tfm));
2763	goto out;
2764	}
2765	if (template[i].setkey_error) {
2766	pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2767	algo, j, template[i].setkey_error);
2768	ret = -EINVAL;
2769	goto out;
2770	}
2771
2772	for (k = 0; k < template[i].len;
2773	k += crypto_cipher_blocksize(tfm)) {
2774	if (enc)
2775	crypto_cipher_encrypt_one(tfm, dst: data + k,
2776	src: data + k);
2777	else
2778	crypto_cipher_decrypt_one(tfm, dst: data + k,
2779	src: data + k);
2780	}
2781
2782	q = data;
2783	if (memcmp(p: q, q: result, size: template[i].len)) {
2784	printk(KERN_ERR "alg: cipher: Test %d failed "
2785	"on %s for %s\n", j, e, algo);
2786	hexdump(buf: q, len: template[i].len);
2787	ret = -EINVAL;
2788	goto out;
2789	}
2790	}
2791
2792	ret = 0;
2793
2794	out:
2795	testmgr_free_buf(buf: xbuf);
2796	out_nobuf:
2797	return ret;
2798	}
2799
2800	static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec,
2801	const char *vec_name,
2802	const struct testvec_config *cfg,
2803	struct skcipher_request *req,
2804	struct cipher_test_sglists *tsgls)
2805	{
2806	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2807	const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2808	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2809	const char *driver = crypto_skcipher_driver_name(tfm);
2810	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2811	const char *op = enc ? "encryption" : "decryption";
2812	DECLARE_CRYPTO_WAIT(wait);
2813	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2814	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2815	cfg->iv_offset +
2816	(cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2817	struct kvec input;
2818	int err;
2819
2820	/* Set the key */
2821	if (vec->wk)
2822	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2823	else
2824	crypto_skcipher_clear_flags(tfm,
2825	CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2826	err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
2827	cfg, alignmask);
2828	if (err) {
2829	if (err == vec->setkey_error)
2830	return 0;
2831	pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2832	driver, vec_name, vec->setkey_error, err,
2833	crypto_skcipher_get_flags(tfm));
2834	return err;
2835	}
2836	if (vec->setkey_error) {
2837	pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2838	driver, vec_name, vec->setkey_error);
2839	return -EINVAL;
2840	}
2841
2842	/* The IV must be copied to a buffer, as the algorithm may modify it */
2843	if (ivsize) {
2844	if (WARN_ON(ivsize > MAX_IVLEN))
2845	return -EINVAL;
2846	if (vec->iv)
2847	memcpy(iv, vec->iv, ivsize);
2848	else
2849	memset(iv, 0, ivsize);
2850	} else {
2851	iv = NULL;
2852	}
2853
2854	/* Build the src/dst scatterlists */
2855	input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2856	input.iov_len = vec->len;
2857	err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2858	src_total_len: vec->len, dst_total_len: vec->len, inputs: &input, nr_inputs: 1);
2859	if (err) {
2860	pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2861	driver, op, vec_name, cfg->name);
2862	return err;
2863	}
2864
2865	/* Do the actual encryption or decryption */
2866	testmgr_poison(addr: req->__ctx, len: crypto_skcipher_reqsize(tfm));
2867	skcipher_request_set_callback(req, flags: req_flags, compl: crypto_req_done, data: &wait);
2868	skcipher_request_set_crypt(req, src: tsgls->src.sgl_ptr, dst: tsgls->dst.sgl_ptr,
2869	cryptlen: vec->len, iv);
2870	if (cfg->nosimd)
2871	crypto_disable_simd_for_test();
2872	err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2873	if (cfg->nosimd)
2874	crypto_reenable_simd_for_test();
2875	err = crypto_wait_req(err, wait: &wait);
2876
2877	/* Check that the algorithm didn't overwrite things it shouldn't have */
2878	if (req->cryptlen != vec->len ||
2879	req->iv != iv ||
2880	req->src != tsgls->src.sgl_ptr ||
2881	req->dst != tsgls->dst.sgl_ptr ||
2882	crypto_skcipher_reqtfm(req) != tfm ||
2883	req->base.complete != crypto_req_done ||
2884	req->base.flags != req_flags ||
2885	req->base.data != &wait) {
2886	pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2887	driver, op, vec_name, cfg->name);
2888	if (req->cryptlen != vec->len)
2889	pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2890	if (req->iv != iv)
2891	pr_err("alg: skcipher: changed 'req->iv'\n");
2892	if (req->src != tsgls->src.sgl_ptr)
2893	pr_err("alg: skcipher: changed 'req->src'\n");
2894	if (req->dst != tsgls->dst.sgl_ptr)
2895	pr_err("alg: skcipher: changed 'req->dst'\n");
2896	if (crypto_skcipher_reqtfm(req) != tfm)
2897	pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2898	if (req->base.complete != crypto_req_done)
2899	pr_err("alg: skcipher: changed 'req->base.complete'\n");
2900	if (req->base.flags != req_flags)
2901	pr_err("alg: skcipher: changed 'req->base.flags'\n");
2902	if (req->base.data != &wait)
2903	pr_err("alg: skcipher: changed 'req->base.data'\n");
2904	return -EINVAL;
2905	}
2906	if (is_test_sglist_corrupted(tsgl: &tsgls->src)) {
2907	pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2908	driver, op, vec_name, cfg->name);
2909	return -EINVAL;
2910	}
2911	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2912	is_test_sglist_corrupted(tsgl: &tsgls->dst)) {
2913	pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2914	driver, op, vec_name, cfg->name);
2915	return -EINVAL;
2916	}
2917
2918	/* Check for success or failure */
2919	if (err) {
2920	if (err == vec->crypt_error)
2921	return 0;
2922	pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2923	driver, op, vec_name, vec->crypt_error, err, cfg->name);
2924	return err;
2925	}
2926	if (vec->crypt_error) {
2927	pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2928	driver, op, vec_name, vec->crypt_error, cfg->name);
2929	return -EINVAL;
2930	}
2931
2932	/* Check for the correct output (ciphertext or plaintext) */
2933	err = verify_correct_output(tsgl: &tsgls->dst, expected_output: enc ? vec->ctext : vec->ptext,
2934	len_to_check: vec->len, unchecked_prefix_len: 0, check_poison: true);
2935	if (err == -EOVERFLOW) {
2936	pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2937	driver, op, vec_name, cfg->name);
2938	return err;
2939	}
2940	if (err) {
2941	pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2942	driver, op, vec_name, cfg->name);
2943	return err;
2944	}
2945
2946	/* If applicable, check that the algorithm generated the correct IV */
2947	if (vec->iv_out && memcmp(p: iv, q: vec->iv_out, size: ivsize) != 0) {
2948	pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2949	driver, op, vec_name, cfg->name);
2950	hexdump(buf: iv, len: ivsize);
2951	return -EINVAL;
2952	}
2953
2954	return 0;
2955	}
2956
2957	static int test_skcipher_vec(int enc, const struct cipher_testvec *vec,
2958	unsigned int vec_num,
2959	struct skcipher_request *req,
2960	struct cipher_test_sglists *tsgls)
2961	{
2962	char vec_name[16];
2963	unsigned int i;
2964	int err;
2965
2966	if (fips_enabled && vec->fips_skip)
2967	return 0;
2968
2969	sprintf(buf: vec_name, fmt: "%u", vec_num);
2970
2971	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2972	err = test_skcipher_vec_cfg(enc, vec, vec_name,
2973	cfg: &default_cipher_testvec_configs[i],
2974	req, tsgls);
2975	if (err)
2976	return err;
2977	}
2978
2979	if (!noslowtests) {
2980	struct rnd_state rng;
2981	struct testvec_config cfg;
2982	char cfgname[TESTVEC_CONFIG_NAMELEN];
2983
2984	init_rnd_state(rng: &rng);
2985
2986	for (i = 0; i < fuzz_iterations; i++) {
2987	generate_random_testvec_config(rng: &rng, cfg: &cfg, name: cfgname,
2988	max_namelen: sizeof(cfgname));
2989	err = test_skcipher_vec_cfg(enc, vec, vec_name,
2990	cfg: &cfg, req, tsgls);
2991	if (err)
2992	return err;
2993	cond_resched();
2994	}
2995	}
2996	return 0;
2997	}
2998
2999	/*
3000	* Generate a symmetric cipher test vector from the given implementation.
3001	* Assumes the buffers in 'vec' were already allocated.
3002	*/
3003	static void generate_random_cipher_testvec(struct rnd_state *rng,
3004	struct skcipher_request *req,
3005	struct cipher_testvec *vec,
3006	unsigned int maxdatasize,
3007	char *name, size_t max_namelen)
3008	{
3009	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3010	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3011	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3012	struct scatterlist src, dst;
3013	u8 iv[MAX_IVLEN];
3014	DECLARE_CRYPTO_WAIT(wait);
3015
3016	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
3017	vec->klen = maxkeysize;
3018	if (prandom_u32_below(rng, ceil: 4) == 0)
3019	vec->klen = prandom_u32_below(rng, ceil: maxkeysize + 1);
3020	generate_random_bytes(rng, buf: (u8 *)vec->key, count: vec->klen);
3021	vec->setkey_error = crypto_skcipher_setkey(tfm, key: vec->key, keylen: vec->klen);
3022
3023	/* IV */
3024	generate_random_bytes(rng, buf: (u8 *)vec->iv, count: ivsize);
3025
3026	/* Plaintext */
3027	vec->len = generate_random_length(rng, max_len: maxdatasize);
3028	generate_random_bytes(rng, buf: (u8 *)vec->ptext, count: vec->len);
3029
3030	/* If the key couldn't be set, no need to continue to encrypt. */
3031	if (vec->setkey_error)
3032	goto done;
3033
3034	/* Ciphertext */
3035	sg_init_one(&src, vec->ptext, vec->len);
3036	sg_init_one(&dst, vec->ctext, vec->len);
3037	memcpy(iv, vec->iv, ivsize);
3038	skcipher_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
3039	skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: vec->len, iv);
3040	vec->crypt_error = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
3041	if (vec->crypt_error != 0) {
3042	/*
3043	* The only acceptable error here is for an invalid length, so
3044	* skcipher decryption should fail with the same error too.
3045	* We'll test for this. But to keep the API usage well-defined,
3046	* explicitly initialize the ciphertext buffer too.
3047	*/
3048	memset((u8 *)vec->ctext, 0, vec->len);
3049	}
3050	done:
3051	snprintf(buf: name, size: max_namelen, fmt: "\"random: len=%u klen=%u\"",
3052	vec->len, vec->klen);
3053	}
3054
3055	/*
3056	* Test the skcipher algorithm represented by @req against the corresponding
3057	* generic implementation, if one is available.
3058	*/
3059	static int test_skcipher_vs_generic_impl(const char *generic_driver,
3060	struct skcipher_request *req,
3061	struct cipher_test_sglists *tsgls)
3062	{
3063	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3064	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3065	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3066	const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
3067	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
3068	const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
3069	const char *driver = crypto_skcipher_driver_name(tfm);
3070	struct rnd_state rng;
3071	char _generic_driver[CRYPTO_MAX_ALG_NAME];
3072	struct crypto_skcipher *generic_tfm = NULL;
3073	struct skcipher_request *generic_req = NULL;
3074	unsigned int i;
3075	struct cipher_testvec vec = { 0 };
3076	char vec_name[64];
3077	struct testvec_config *cfg;
3078	char cfgname[TESTVEC_CONFIG_NAMELEN];
3079	int err;
3080
3081	if (noslowtests)
3082	return 0;
3083
3084	init_rnd_state(rng: &rng);
3085
3086	if (!generic_driver) { /* Use default naming convention? */
3087	err = build_generic_driver_name(algname, driver_name: _generic_driver);
3088	if (err)
3089	return err;
3090	generic_driver = _generic_driver;
3091	}
3092
3093	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
3094	return 0;
3095
3096	generic_tfm = crypto_alloc_skcipher(alg_name: generic_driver, type: 0, mask: 0);
3097	if (IS_ERR(ptr: generic_tfm)) {
3098	err = PTR_ERR(ptr: generic_tfm);
3099	if (err == -ENOENT) {
3100	pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
3101	driver, generic_driver);
3102	return 0;
3103	}
3104	pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
3105	generic_driver, algname, err);
3106	return err;
3107	}
3108
3109	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
3110	if (!cfg) {
3111	err = -ENOMEM;
3112	goto out;
3113	}
3114
3115	generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
3116	if (!generic_req) {
3117	err = -ENOMEM;
3118	goto out;
3119	}
3120
3121	/* Check the algorithm properties for consistency. */
3122
3123	if (crypto_skcipher_min_keysize(tfm) !=
3124	crypto_skcipher_min_keysize(tfm: generic_tfm)) {
3125	pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
3126	driver, crypto_skcipher_min_keysize(tfm),
3127	crypto_skcipher_min_keysize(generic_tfm));
3128	err = -EINVAL;
3129	goto out;
3130	}
3131
3132	if (maxkeysize != crypto_skcipher_max_keysize(tfm: generic_tfm)) {
3133	pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
3134	driver, maxkeysize,
3135	crypto_skcipher_max_keysize(generic_tfm));
3136	err = -EINVAL;
3137	goto out;
3138	}
3139
3140	if (ivsize != crypto_skcipher_ivsize(tfm: generic_tfm)) {
3141	pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
3142	driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
3143	err = -EINVAL;
3144	goto out;
3145	}
3146
3147	if (blocksize != crypto_skcipher_blocksize(tfm: generic_tfm)) {
3148	pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
3149	driver, blocksize,
3150	crypto_skcipher_blocksize(generic_tfm));
3151	err = -EINVAL;
3152	goto out;
3153	}
3154
3155	/*
3156	* Now generate test vectors using the generic implementation, and test
3157	* the other implementation against them.
3158	*/
3159
3160	vec.key = kmalloc(maxkeysize, GFP_KERNEL);
3161	vec.iv = kmalloc(ivsize, GFP_KERNEL);
3162	vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
3163	vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
3164	if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
3165	err = -ENOMEM;
3166	goto out;
3167	}
3168
3169	for (i = 0; i < fuzz_iterations * 8; i++) {
3170	generate_random_cipher_testvec(rng: &rng, req: generic_req, vec: &vec,
3171	maxdatasize,
3172	name: vec_name, max_namelen: sizeof(vec_name));
3173	generate_random_testvec_config(rng: &rng, cfg, name: cfgname,
3174	max_namelen: sizeof(cfgname));
3175
3176	err = test_skcipher_vec_cfg(ENCRYPT, vec: &vec, vec_name,
3177	cfg, req, tsgls);
3178	if (err)
3179	goto out;
3180	err = test_skcipher_vec_cfg(DECRYPT, vec: &vec, vec_name,
3181	cfg, req, tsgls);
3182	if (err)
3183	goto out;
3184	cond_resched();
3185	}
3186	err = 0;
3187	out:
3188	kfree(objp: cfg);
3189	kfree(objp: vec.key);
3190	kfree(objp: vec.iv);
3191	kfree(objp: vec.ptext);
3192	kfree(objp: vec.ctext);
3193	crypto_free_skcipher(tfm: generic_tfm);
3194	skcipher_request_free(req: generic_req);
3195	return err;
3196	}
3197
3198	static int test_skcipher(int enc, const struct cipher_test_suite *suite,
3199	struct skcipher_request *req,
3200	struct cipher_test_sglists *tsgls)
3201	{
3202	unsigned int i;
3203	int err;
3204
3205	for (i = 0; i < suite->count; i++) {
3206	err = test_skcipher_vec(enc, vec: &suite->vecs[i], vec_num: i, req, tsgls);
3207	if (err)
3208	return err;
3209	cond_resched();
3210	}
3211	return 0;
3212	}
3213
3214	static int alg_test_skcipher(const struct alg_test_desc *desc,
3215	const char *driver, u32 type, u32 mask)
3216	{
3217	const struct cipher_test_suite *suite = &desc->suite.cipher;
3218	struct crypto_skcipher *tfm;
3219	struct skcipher_request *req = NULL;
3220	struct cipher_test_sglists *tsgls = NULL;
3221	int err;
3222
3223	if (suite->count <= 0) {
3224	pr_err("alg: skcipher: empty test suite for %s\n", driver);
3225	return -EINVAL;
3226	}
3227
3228	tfm = crypto_alloc_skcipher(alg_name: driver, type, mask);
3229	if (IS_ERR(ptr: tfm)) {
3230	if (PTR_ERR(ptr: tfm) == -ENOENT)
3231	return 0;
3232	pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
3233	driver, PTR_ERR(tfm));
3234	return PTR_ERR(ptr: tfm);
3235	}
3236	driver = crypto_skcipher_driver_name(tfm);
3237
3238	req = skcipher_request_alloc(tfm, GFP_KERNEL);
3239	if (!req) {
3240	pr_err("alg: skcipher: failed to allocate request for %s\n",
3241	driver);
3242	err = -ENOMEM;
3243	goto out;
3244	}
3245
3246	tsgls = alloc_cipher_test_sglists();
3247	if (!tsgls) {
3248	pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
3249	driver);
3250	err = -ENOMEM;
3251	goto out;
3252	}
3253
3254	err = test_skcipher(ENCRYPT, suite, req, tsgls);
3255	if (err)
3256	goto out;
3257
3258	err = test_skcipher(DECRYPT, suite, req, tsgls);
3259	if (err)
3260	goto out;
3261
3262	err = test_skcipher_vs_generic_impl(generic_driver: desc->generic_driver, req, tsgls);
3263	out:
3264	free_cipher_test_sglists(tsgls);
3265	skcipher_request_free(req);
3266	crypto_free_skcipher(tfm);
3267	return err;
3268	}
3269
3270	static int test_acomp(struct crypto_acomp *tfm,
3271	const struct comp_testvec *ctemplate,
3272	const struct comp_testvec *dtemplate,
3273	int ctcount, int dtcount)
3274	{
3275	const char *algo = crypto_tfm_alg_driver_name(tfm: crypto_acomp_tfm(tfm));
3276	unsigned int i;
3277	char *output, *decomp_out;
3278	int ret;
3279	struct scatterlist src, dst;
3280	struct acomp_req *req;
3281	struct crypto_wait wait;
3282
3283	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3284	if (!output)
3285	return -ENOMEM;
3286
3287	decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3288	if (!decomp_out) {
3289	kfree(objp: output);
3290	return -ENOMEM;
3291	}
3292
3293	for (i = 0; i < ctcount; i++) {
3294	unsigned int dlen = COMP_BUF_SIZE;
3295	int ilen = ctemplate[i].inlen;
3296	void *input_vec;
3297
3298	input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
3299	if (!input_vec) {
3300	ret = -ENOMEM;
3301	goto out;
3302	}
3303
3304	memset(output, 0, dlen);
3305	crypto_init_wait(wait: &wait);
3306	sg_init_one(&src, input_vec, ilen);
3307	sg_init_one(&dst, output, dlen);
3308
3309	req = acomp_request_alloc(tfm);
3310	if (!req) {
3311	pr_err("alg: acomp: request alloc failed for %s\n",
3312	algo);
3313	kfree(objp: input_vec);
3314	ret = -ENOMEM;
3315	goto out;
3316	}
3317
3318	acomp_request_set_params(req, src: &src, dst: &dst, slen: ilen, dlen);
3319	acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3320	cmpl: crypto_req_done, data: &wait);
3321
3322	ret = crypto_wait_req(err: crypto_acomp_compress(req), wait: &wait);
3323	if (ret) {
3324	pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3325	i + 1, algo, -ret);
3326	kfree(objp: input_vec);
3327	acomp_request_free(req);
3328	goto out;
3329	}
3330
3331	ilen = req->dlen;
3332	dlen = COMP_BUF_SIZE;
3333	sg_init_one(&src, output, ilen);
3334	sg_init_one(&dst, decomp_out, dlen);
3335	crypto_init_wait(wait: &wait);
3336	acomp_request_set_params(req, src: &src, dst: &dst, slen: ilen, dlen);
3337
3338	ret = crypto_wait_req(err: crypto_acomp_decompress(req), wait: &wait);
3339	if (ret) {
3340	pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3341	i + 1, algo, -ret);
3342	kfree(objp: input_vec);
3343	acomp_request_free(req);
3344	goto out;
3345	}
3346
3347	if (req->dlen != ctemplate[i].inlen) {
3348	pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
3349	i + 1, algo, req->dlen);
3350	ret = -EINVAL;
3351	kfree(objp: input_vec);
3352	acomp_request_free(req);
3353	goto out;
3354	}
3355
3356	if (memcmp(p: input_vec, q: decomp_out, size: req->dlen)) {
3357	pr_err("alg: acomp: Compression test %d failed for %s\n",
3358	i + 1, algo);
3359	hexdump(buf: output, len: req->dlen);
3360	ret = -EINVAL;
3361	kfree(objp: input_vec);
3362	acomp_request_free(req);
3363	goto out;
3364	}
3365
3366	kfree(objp: input_vec);
3367	acomp_request_free(req);
3368	}
3369
3370	for (i = 0; i < dtcount; i++) {
3371	unsigned int dlen = COMP_BUF_SIZE;
3372	int ilen = dtemplate[i].inlen;
3373	void *input_vec;
3374
3375	input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
3376	if (!input_vec) {
3377	ret = -ENOMEM;
3378	goto out;
3379	}
3380
3381	memset(output, 0, dlen);
3382	crypto_init_wait(wait: &wait);
3383	sg_init_one(&src, input_vec, ilen);
3384	sg_init_one(&dst, output, dlen);
3385
3386	req = acomp_request_alloc(tfm);
3387	if (!req) {
3388	pr_err("alg: acomp: request alloc failed for %s\n",
3389	algo);
3390	kfree(objp: input_vec);
3391	ret = -ENOMEM;
3392	goto out;
3393	}
3394
3395	acomp_request_set_params(req, src: &src, dst: &dst, slen: ilen, dlen);
3396	acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3397	cmpl: crypto_req_done, data: &wait);
3398
3399	ret = crypto_wait_req(err: crypto_acomp_decompress(req), wait: &wait);
3400	if (ret) {
3401	pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
3402	i + 1, algo, -ret);
3403	kfree(objp: input_vec);
3404	acomp_request_free(req);
3405	goto out;
3406	}
3407
3408	if (req->dlen != dtemplate[i].outlen) {
3409	pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
3410	i + 1, algo, req->dlen);
3411	ret = -EINVAL;
3412	kfree(objp: input_vec);
3413	acomp_request_free(req);
3414	goto out;
3415	}
3416
3417	if (memcmp(p: output, q: dtemplate[i].output, size: req->dlen)) {
3418	pr_err("alg: acomp: Decompression test %d failed for %s\n",
3419	i + 1, algo);
3420	hexdump(buf: output, len: req->dlen);
3421	ret = -EINVAL;
3422	kfree(objp: input_vec);
3423	acomp_request_free(req);
3424	goto out;
3425	}
3426
3427	kfree(objp: input_vec);
3428	acomp_request_free(req);
3429	}
3430
3431	ret = 0;
3432
3433	out:
3434	kfree(objp: decomp_out);
3435	kfree(objp: output);
3436	return ret;
3437	}
3438
3439	static int alg_test_cipher(const struct alg_test_desc *desc,
3440	const char *driver, u32 type, u32 mask)
3441	{
3442	const struct cipher_test_suite *suite = &desc->suite.cipher;
3443	struct crypto_cipher *tfm;
3444	int err;
3445
3446	tfm = crypto_alloc_cipher(alg_name: driver, type, mask);
3447	if (IS_ERR(ptr: tfm)) {
3448	if (PTR_ERR(ptr: tfm) == -ENOENT)
3449	return 0;
3450	printk(KERN_ERR "alg: cipher: Failed to load transform for "
3451	"%s: %ld\n", driver, PTR_ERR(tfm));
3452	return PTR_ERR(ptr: tfm);
3453	}
3454
3455	err = test_cipher(tfm, ENCRYPT, template: suite->vecs, tcount: suite->count);
3456	if (!err)
3457	err = test_cipher(tfm, DECRYPT, template: suite->vecs, tcount: suite->count);
3458
3459	crypto_free_cipher(tfm);
3460	return err;
3461	}
3462
3463	static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
3464	u32 type, u32 mask)
3465	{
3466	struct crypto_acomp *acomp;
3467	int err;
3468
3469	acomp = crypto_alloc_acomp(alg_name: driver, type, mask);
3470	if (IS_ERR(ptr: acomp)) {
3471	if (PTR_ERR(ptr: acomp) == -ENOENT)
3472	return 0;
3473	pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
3474	driver, PTR_ERR(acomp));
3475	return PTR_ERR(ptr: acomp);
3476	}
3477	err = test_acomp(tfm: acomp, ctemplate: desc->suite.comp.comp.vecs,
3478	dtemplate: desc->suite.comp.decomp.vecs,
3479	ctcount: desc->suite.comp.comp.count,
3480	dtcount: desc->suite.comp.decomp.count);
3481	crypto_free_acomp(tfm: acomp);
3482	return err;
3483	}
3484
3485	static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3486	const char *driver, u32 type, u32 mask)
3487	{
3488	int ret = -EAGAIN;
3489	struct crypto_rng *drng;
3490	struct drbg_test_data test_data;
3491	struct drbg_string addtl, pers, testentropy;
3492	unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3493
3494	if (!buf)
3495	return -ENOMEM;
3496
3497	drng = crypto_alloc_rng(alg_name: driver, type, mask);
3498	if (IS_ERR(ptr: drng)) {
3499	kfree_sensitive(objp: buf);
3500	if (PTR_ERR(ptr: drng) == -ENOENT)
3501	return 0;
3502	printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3503	"%s\n", driver);
3504	return PTR_ERR(ptr: drng);
3505	}
3506
3507	test_data.testentropy = &testentropy;
3508	drbg_string_fill(string: &testentropy, buf: test->entropy, len: test->entropylen);
3509	drbg_string_fill(string: &pers, buf: test->pers, len: test->perslen);
3510	ret = crypto_drbg_reset_test(drng, pers: &pers, test_data: &test_data);
3511	if (ret) {
3512	printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3513	goto outbuf;
3514	}
3515
3516	drbg_string_fill(string: &addtl, buf: test->addtla, len: test->addtllen);
3517	if (pr) {
3518	drbg_string_fill(string: &testentropy, buf: test->entpra, len: test->entprlen);
3519	ret = crypto_drbg_get_bytes_addtl_test(drng,
3520	outbuf: buf, outlen: test->expectedlen, addtl: &addtl, test_data: &test_data);
3521	} else {
3522	ret = crypto_drbg_get_bytes_addtl(drng,
3523	outbuf: buf, outlen: test->expectedlen, addtl: &addtl);
3524	}
3525	if (ret < 0) {
3526	printk(KERN_ERR "alg: drbg: could not obtain random data for "
3527	"driver %s\n", driver);
3528	goto outbuf;
3529	}
3530
3531	drbg_string_fill(string: &addtl, buf: test->addtlb, len: test->addtllen);
3532	if (pr) {
3533	drbg_string_fill(string: &testentropy, buf: test->entprb, len: test->entprlen);
3534	ret = crypto_drbg_get_bytes_addtl_test(drng,
3535	outbuf: buf, outlen: test->expectedlen, addtl: &addtl, test_data: &test_data);
3536	} else {
3537	ret = crypto_drbg_get_bytes_addtl(drng,
3538	outbuf: buf, outlen: test->expectedlen, addtl: &addtl);
3539	}
3540	if (ret < 0) {
3541	printk(KERN_ERR "alg: drbg: could not obtain random data for "
3542	"driver %s\n", driver);
3543	goto outbuf;
3544	}
3545
3546	ret = memcmp(p: test->expected, q: buf, size: test->expectedlen);
3547
3548	outbuf:
3549	crypto_free_rng(tfm: drng);
3550	kfree_sensitive(objp: buf);
3551	return ret;
3552	}
3553
3554
3555	static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3556	u32 type, u32 mask)
3557	{
3558	int err = 0;
3559	int pr = 0;
3560	int i = 0;
3561	const struct drbg_testvec *template = desc->suite.drbg.vecs;
3562	unsigned int tcount = desc->suite.drbg.count;
3563
3564	if (0 == memcmp(p: driver, q: "drbg_pr_", size: 8))
3565	pr = 1;
3566
3567	for (i = 0; i < tcount; i++) {
3568	err = drbg_cavs_test(test: &template[i], pr, driver, type, mask);
3569	if (err) {
3570	printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3571	i, driver);
3572	err = -EINVAL;
3573	break;
3574	}
3575	}
3576	return err;
3577
3578	}
3579
3580	static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3581	const char *alg)
3582	{
3583	struct kpp_request *req;
3584	void *input_buf = NULL;
3585	void *output_buf = NULL;
3586	void *a_public = NULL;
3587	void *a_ss = NULL;
3588	void *shared_secret = NULL;
3589	struct crypto_wait wait;
3590	unsigned int out_len_max;
3591	int err = -ENOMEM;
3592	struct scatterlist src, dst;
3593
3594	req = kpp_request_alloc(tfm, GFP_KERNEL);
3595	if (!req)
3596	return err;
3597
3598	crypto_init_wait(wait: &wait);
3599
3600	err = crypto_kpp_set_secret(tfm, buffer: vec->secret, len: vec->secret_size);
3601	if (err < 0)
3602	goto free_req;
3603
3604	out_len_max = crypto_kpp_maxsize(tfm);
3605	output_buf = kzalloc(out_len_max, GFP_KERNEL);
3606	if (!output_buf) {
3607	err = -ENOMEM;
3608	goto free_req;
3609	}
3610
3611	/* Use appropriate parameter as base */
3612	kpp_request_set_input(req, NULL, input_len: 0);
3613	sg_init_one(&dst, output_buf, out_len_max);
3614	kpp_request_set_output(req, output: &dst, output_len: out_len_max);
3615	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3616	cmpl: crypto_req_done, data: &wait);
3617
3618	/* Compute party A's public key */
3619	err = crypto_wait_req(err: crypto_kpp_generate_public_key(req), wait: &wait);
3620	if (err) {
3621	pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3622	alg, err);
3623	goto free_output;
3624	}
3625
3626	if (vec->genkey) {
3627	/* Save party A's public key */
3628	a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3629	if (!a_public) {
3630	err = -ENOMEM;
3631	goto free_output;
3632	}
3633	} else {
3634	/* Verify calculated public key */
3635	if (memcmp(p: vec->expected_a_public, q: sg_virt(sg: req->dst),
3636	size: vec->expected_a_public_size)) {
3637	pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3638	alg);
3639	err = -EINVAL;
3640	goto free_output;
3641	}
3642	}
3643
3644	/* Calculate shared secret key by using counter part (b) public key. */
3645	input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
3646	if (!input_buf) {
3647	err = -ENOMEM;
3648	goto free_output;
3649	}
3650
3651	sg_init_one(&src, input_buf, vec->b_public_size);
3652	sg_init_one(&dst, output_buf, out_len_max);
3653	kpp_request_set_input(req, input: &src, input_len: vec->b_public_size);
3654	kpp_request_set_output(req, output: &dst, output_len: out_len_max);
3655	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3656	cmpl: crypto_req_done, data: &wait);
3657	err = crypto_wait_req(err: crypto_kpp_compute_shared_secret(req), wait: &wait);
3658	if (err) {
3659	pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
3660	alg, err);
3661	goto free_all;
3662	}
3663
3664	if (vec->genkey) {
3665	/* Save the shared secret obtained by party A */
3666	a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
3667	if (!a_ss) {
3668	err = -ENOMEM;
3669	goto free_all;
3670	}
3671
3672	/*
3673	* Calculate party B's shared secret by using party A's
3674	* public key.
3675	*/
3676	err = crypto_kpp_set_secret(tfm, buffer: vec->b_secret,
3677	len: vec->b_secret_size);
3678	if (err < 0)
3679	goto free_all;
3680
3681	sg_init_one(&src, a_public, vec->expected_a_public_size);
3682	sg_init_one(&dst, output_buf, out_len_max);
3683	kpp_request_set_input(req, input: &src, input_len: vec->expected_a_public_size);
3684	kpp_request_set_output(req, output: &dst, output_len: out_len_max);
3685	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3686	cmpl: crypto_req_done, data: &wait);
3687	err = crypto_wait_req(err: crypto_kpp_compute_shared_secret(req),
3688	wait: &wait);
3689	if (err) {
3690	pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
3691	alg, err);
3692	goto free_all;
3693	}
3694
3695	shared_secret = a_ss;
3696	} else {
3697	shared_secret = (void *)vec->expected_ss;
3698	}
3699
3700	/*
3701	* verify shared secret from which the user will derive
3702	* secret key by executing whatever hash it has chosen
3703	*/
3704	if (memcmp(p: shared_secret, q: sg_virt(sg: req->dst),
3705	size: vec->expected_ss_size)) {
3706	pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
3707	alg);
3708	err = -EINVAL;
3709	}
3710
3711	free_all:
3712	kfree(objp: a_ss);
3713	kfree(objp: input_buf);
3714	free_output:
3715	kfree(objp: a_public);
3716	kfree(objp: output_buf);
3717	free_req:
3718	kpp_request_free(req);
3719	return err;
3720	}
3721
3722	static int test_kpp(struct crypto_kpp *tfm, const char *alg,
3723	const struct kpp_testvec *vecs, unsigned int tcount)
3724	{
3725	int ret, i;
3726
3727	for (i = 0; i < tcount; i++) {
3728	ret = do_test_kpp(tfm, vec: vecs++, alg);
3729	if (ret) {
3730	pr_err("alg: %s: test failed on vector %d, err=%d\n",
3731	alg, i + 1, ret);
3732	return ret;
3733	}
3734	}
3735	return 0;
3736	}
3737
3738	static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
3739	u32 type, u32 mask)
3740	{
3741	struct crypto_kpp *tfm;
3742	int err = 0;
3743
3744	tfm = crypto_alloc_kpp(alg_name: driver, type, mask);
3745	if (IS_ERR(ptr: tfm)) {
3746	if (PTR_ERR(ptr: tfm) == -ENOENT)
3747	return 0;
3748	pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
3749	driver, PTR_ERR(tfm));
3750	return PTR_ERR(ptr: tfm);
3751	}
3752	if (desc->suite.kpp.vecs)
3753	err = test_kpp(tfm, alg: desc->alg, vecs: desc->suite.kpp.vecs,
3754	tcount: desc->suite.kpp.count);
3755
3756	crypto_free_kpp(tfm);
3757	return err;
3758	}
3759
3760	static u8 *test_pack_u32(u8 *dst, u32 val)
3761	{
3762	memcpy(dst, &val, sizeof(val));
3763	return dst + sizeof(val);
3764	}
3765
3766	static int test_akcipher_one(struct crypto_akcipher *tfm,
3767	const struct akcipher_testvec *vecs)
3768	{
3769	char *xbuf[XBUFSIZE];
3770	struct akcipher_request *req;
3771	void *outbuf_enc = NULL;
3772	void *outbuf_dec = NULL;
3773	struct crypto_wait wait;
3774	unsigned int out_len_max, out_len = 0;
3775	int err = -ENOMEM;
3776	struct scatterlist src, dst, src_tab[2];
3777	const char *c;
3778	unsigned int c_size;
3779
3780	if (testmgr_alloc_buf(buf: xbuf))
3781	return err;
3782
3783	req = akcipher_request_alloc(tfm, GFP_KERNEL);
3784	if (!req)
3785	goto free_xbuf;
3786
3787	crypto_init_wait(wait: &wait);
3788
3789	if (vecs->public_key_vec)
3790	err = crypto_akcipher_set_pub_key(tfm, key: vecs->key,
3791	keylen: vecs->key_len);
3792	else
3793	err = crypto_akcipher_set_priv_key(tfm, key: vecs->key,
3794	keylen: vecs->key_len);
3795	if (err)
3796	goto free_req;
3797
3798	/* First run encrypt test which does not require a private key */
3799	err = -ENOMEM;
3800	out_len_max = crypto_akcipher_maxsize(tfm);
3801	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
3802	if (!outbuf_enc)
3803	goto free_req;
3804
3805	c = vecs->c;
3806	c_size = vecs->c_size;
3807
3808	err = -E2BIG;
3809	if (WARN_ON(vecs->m_size > PAGE_SIZE))
3810	goto free_all;
3811	memcpy(xbuf[0], vecs->m, vecs->m_size);
3812
3813	sg_init_table(src_tab, 2);
3814	sg_set_buf(sg: &src_tab[0], buf: xbuf[0], buflen: 8);
3815	sg_set_buf(sg: &src_tab[1], buf: xbuf[0] + 8, buflen: vecs->m_size - 8);
3816	sg_init_one(&dst, outbuf_enc, out_len_max);
3817	akcipher_request_set_crypt(req, src: src_tab, dst: &dst, src_len: vecs->m_size,
3818	dst_len: out_len_max);
3819	akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3820	cmpl: crypto_req_done, data: &wait);
3821
3822	err = crypto_wait_req(err: crypto_akcipher_encrypt(req), wait: &wait);
3823	if (err) {
3824	pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
3825	goto free_all;
3826	}
3827	if (c) {
3828	if (req->dst_len != c_size) {
3829	pr_err("alg: akcipher: encrypt test failed. Invalid output len\n");
3830	err = -EINVAL;
3831	goto free_all;
3832	}
3833	/* verify that encrypted message is equal to expected */
3834	if (memcmp(p: c, q: outbuf_enc, size: c_size) != 0) {
3835	pr_err("alg: akcipher: encrypt test failed. Invalid output\n");
3836	hexdump(buf: outbuf_enc, len: c_size);
3837	err = -EINVAL;
3838	goto free_all;
3839	}
3840	}
3841
3842	/*
3843	* Don't invoke decrypt test which requires a private key
3844	* for vectors with only a public key.
3845	*/
3846	if (vecs->public_key_vec) {
3847	err = 0;
3848	goto free_all;
3849	}
3850	outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
3851	if (!outbuf_dec) {
3852	err = -ENOMEM;
3853	goto free_all;
3854	}
3855
3856	if (!c) {
3857	c = outbuf_enc;
3858	c_size = req->dst_len;
3859	}
3860
3861	err = -E2BIG;
3862	if (WARN_ON(c_size > PAGE_SIZE))
3863	goto free_all;
3864	memcpy(xbuf[0], c, c_size);
3865
3866	sg_init_one(&src, xbuf[0], c_size);
3867	sg_init_one(&dst, outbuf_dec, out_len_max);
3868	crypto_init_wait(wait: &wait);
3869	akcipher_request_set_crypt(req, src: &src, dst: &dst, src_len: c_size, dst_len: out_len_max);
3870
3871	err = crypto_wait_req(err: crypto_akcipher_decrypt(req), wait: &wait);
3872	if (err) {
3873	pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
3874	goto free_all;
3875	}
3876	out_len = req->dst_len;
3877	if (out_len < vecs->m_size) {
3878	pr_err("alg: akcipher: decrypt test failed. Invalid output len %u\n",
3879	out_len);
3880	err = -EINVAL;
3881	goto free_all;
3882	}
3883	/* verify that decrypted message is equal to the original msg */
3884	if (memchr_inv(p: outbuf_dec, c: 0, size: out_len - vecs->m_size) ||
3885	memcmp(p: vecs->m, q: outbuf_dec + out_len - vecs->m_size,
3886	size: vecs->m_size)) {
3887	pr_err("alg: akcipher: decrypt test failed. Invalid output\n");
3888	hexdump(buf: outbuf_dec, len: out_len);
3889	err = -EINVAL;
3890	}
3891	free_all:
3892	kfree(objp: outbuf_dec);
3893	kfree(objp: outbuf_enc);
3894	free_req:
3895	akcipher_request_free(req);
3896	free_xbuf:
3897	testmgr_free_buf(buf: xbuf);
3898	return err;
3899	}
3900
3901	static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
3902	const struct akcipher_testvec *vecs,
3903	unsigned int tcount)
3904	{
3905	const char *algo =
3906	crypto_tfm_alg_driver_name(tfm: crypto_akcipher_tfm(tfm));
3907	int ret, i;
3908
3909	for (i = 0; i < tcount; i++) {
3910	ret = test_akcipher_one(tfm, vecs: vecs++);
3911	if (!ret)
3912	continue;
3913
3914	pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
3915	i + 1, algo, ret);
3916	return ret;
3917	}
3918	return 0;
3919	}
3920
3921	static int alg_test_akcipher(const struct alg_test_desc *desc,
3922	const char *driver, u32 type, u32 mask)
3923	{
3924	struct crypto_akcipher *tfm;
3925	int err = 0;
3926
3927	tfm = crypto_alloc_akcipher(alg_name: driver, type, mask);
3928	if (IS_ERR(ptr: tfm)) {
3929	if (PTR_ERR(ptr: tfm) == -ENOENT)
3930	return 0;
3931	pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
3932	driver, PTR_ERR(tfm));
3933	return PTR_ERR(ptr: tfm);
3934	}
3935	if (desc->suite.akcipher.vecs)
3936	err = test_akcipher(tfm, alg: desc->alg, vecs: desc->suite.akcipher.vecs,
3937	tcount: desc->suite.akcipher.count);
3938
3939	crypto_free_akcipher(tfm);
3940	return err;
3941	}
3942
3943	static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs)
3944	{
3945	u8 *ptr, *key __free(kfree);
3946	int err, sig_size;
3947
3948	key = kmalloc(vecs->key_len + 2 * sizeof(u32) + vecs->param_len,
3949	GFP_KERNEL);
3950	if (!key)
3951	return -ENOMEM;
3952
3953	/* ecrdsa expects additional parameters appended to the key */
3954	memcpy(key, vecs->key, vecs->key_len);
3955	ptr = key + vecs->key_len;
3956	ptr = test_pack_u32(dst: ptr, val: vecs->algo);
3957	ptr = test_pack_u32(dst: ptr, val: vecs->param_len);
3958	memcpy(ptr, vecs->params, vecs->param_len);
3959
3960	if (vecs->public_key_vec)
3961	err = crypto_sig_set_pubkey(tfm, key, keylen: vecs->key_len);
3962	else
3963	err = crypto_sig_set_privkey(tfm, key, keylen: vecs->key_len);
3964	if (err)
3965	return err;
3966
3967	/*
3968	* Run asymmetric signature verification first
3969	* (which does not require a private key)
3970	*/
3971	err = crypto_sig_verify(tfm, src: vecs->c, slen: vecs->c_size,
3972	digest: vecs->m, dlen: vecs->m_size);
3973	if (err) {
3974	pr_err("alg: sig: verify test failed: err %d\n", err);
3975	return err;
3976	}
3977
3978	/*
3979	* Don't invoke sign test (which requires a private key)
3980	* for vectors with only a public key.
3981	*/
3982	if (vecs->public_key_vec)
3983	return 0;
3984
3985	sig_size = crypto_sig_maxsize(tfm);
3986	if (sig_size < vecs->c_size) {
3987	pr_err("alg: sig: invalid maxsize %u\n", sig_size);
3988	return -EINVAL;
3989	}
3990
3991	u8 *sig __free(kfree) = kzalloc(sig_size, GFP_KERNEL);
3992	if (!sig)
3993	return -ENOMEM;
3994
3995	/* Run asymmetric signature generation */
3996	err = crypto_sig_sign(tfm, src: vecs->m, slen: vecs->m_size, dst: sig, dlen: sig_size);
3997	if (err < 0) {
3998	pr_err("alg: sig: sign test failed: err %d\n", err);
3999	return err;
4000	}
4001
4002	/* Verify that generated signature equals cooked signature */
4003	if (err != vecs->c_size ||
4004	memcmp(p: sig, q: vecs->c, size: vecs->c_size) ||
4005	memchr_inv(p: sig + vecs->c_size, c: 0, size: sig_size - vecs->c_size)) {
4006	pr_err("alg: sig: sign test failed: invalid output\n");
4007	hexdump(buf: sig, len: sig_size);
4008	return -EINVAL;
4009	}
4010
4011	return 0;
4012	}
4013
4014	static int test_sig(struct crypto_sig *tfm, const char *alg,
4015	const struct sig_testvec *vecs, unsigned int tcount)
4016	{
4017	const char *algo = crypto_tfm_alg_driver_name(tfm: crypto_sig_tfm(tfm));
4018	int ret, i;
4019
4020	for (i = 0; i < tcount; i++) {
4021	ret = test_sig_one(tfm, vecs: vecs++);
4022	if (ret) {
4023	pr_err("alg: sig: test %d failed for %s: err %d\n",
4024	i + 1, algo, ret);
4025	return ret;
4026	}
4027	}
4028	return 0;
4029	}
4030
4031	static int alg_test_sig(const struct alg_test_desc *desc, const char *driver,
4032	u32 type, u32 mask)
4033	{
4034	struct crypto_sig *tfm;
4035	int err = 0;
4036
4037	tfm = crypto_alloc_sig(alg_name: driver, type, mask);
4038	if (IS_ERR(ptr: tfm)) {
4039	pr_err("alg: sig: Failed to load tfm for %s: %ld\n",
4040	driver, PTR_ERR(tfm));
4041	return PTR_ERR(ptr: tfm);
4042	}
4043	if (desc->suite.sig.vecs)
4044	err = test_sig(tfm, alg: desc->alg, vecs: desc->suite.sig.vecs,
4045	tcount: desc->suite.sig.count);
4046
4047	crypto_free_sig(tfm);
4048	return err;
4049	}
4050
4051	static int alg_test_null(const struct alg_test_desc *desc,
4052	const char *driver, u32 type, u32 mask)
4053	{
4054	return 0;
4055	}
4056
4057	#define ____VECS(tv) .vecs = tv, .count = ARRAY_SIZE(tv)
4058	#define __VECS(tv) { ____VECS(tv) }
4059
4060	/* Please keep this list sorted by algorithm name. */
4061	static const struct alg_test_desc alg_test_descs[] = {
4062	{
4063	.alg = "adiantum(xchacha12,aes)",
4064	.generic_driver = "adiantum(xchacha12-lib,aes-generic,nhpoly1305-generic)",
4065	.test = alg_test_skcipher,
4066	.suite = {
4067	.cipher = __VECS(adiantum_xchacha12_aes_tv_template)
4068	},
4069	}, {
4070	.alg = "adiantum(xchacha20,aes)",
4071	.generic_driver = "adiantum(xchacha20-lib,aes-generic,nhpoly1305-generic)",
4072	.test = alg_test_skcipher,
4073	.suite = {
4074	.cipher = __VECS(adiantum_xchacha20_aes_tv_template)
4075	},
4076	}, {
4077	.alg = "aegis128",
4078	.test = alg_test_aead,
4079	.suite = {
4080	.aead = __VECS(aegis128_tv_template)
4081	}
4082	}, {
4083	.alg = "authenc(hmac(md5),ecb(cipher_null))",
4084	.generic_driver = "authenc(hmac-md5-lib,ecb-cipher_null)",
4085	.test = alg_test_aead,
4086	.suite = {
4087	.aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
4088	}
4089	}, {
4090	.alg = "authenc(hmac(sha1),cbc(aes))",
4091	.generic_driver = "authenc(hmac-sha1-lib,cbc(aes-generic))",
4092	.test = alg_test_aead,
4093	.fips_allowed = 1,
4094	.suite = {
4095	.aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4096	}
4097	}, {
4098	.alg = "authenc(hmac(sha1),cbc(des))",
4099	.generic_driver = "authenc(hmac-sha1-lib,cbc(des-generic))",
4100	.test = alg_test_aead,
4101	.suite = {
4102	.aead = __VECS(hmac_sha1_des_cbc_tv_temp)
4103	}
4104	}, {
4105	.alg = "authenc(hmac(sha1),cbc(des3_ede))",
4106	.generic_driver = "authenc(hmac-sha1-lib,cbc(des3_ede-generic))",
4107	.test = alg_test_aead,
4108	.suite = {
4109	.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
4110	}
4111	}, {
4112	.alg = "authenc(hmac(sha1),ctr(aes))",
4113	.test = alg_test_null,
4114	.fips_allowed = 1,
4115	}, {
4116	.alg = "authenc(hmac(sha1),ecb(cipher_null))",
4117	.generic_driver = "authenc(hmac-sha1-lib,ecb-cipher_null)",
4118	.test = alg_test_aead,
4119	.suite = {
4120	.aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
4121	}
4122	}, {
4123	.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4124	.test = alg_test_null,
4125	.fips_allowed = 1,
4126	}, {
4127	.alg = "authenc(hmac(sha224),cbc(des))",
4128	.generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))",
4129	.test = alg_test_aead,
4130	.suite = {
4131	.aead = __VECS(hmac_sha224_des_cbc_tv_temp)
4132	}
4133	}, {
4134	.alg = "authenc(hmac(sha224),cbc(des3_ede))",
4135	.generic_driver = "authenc(hmac-sha224-lib,cbc(des3_ede-generic))",
4136	.test = alg_test_aead,
4137	.suite = {
4138	.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
4139	}
4140	}, {
4141	.alg = "authenc(hmac(sha256),cbc(aes))",
4142	.generic_driver = "authenc(hmac-sha256-lib,cbc(aes-generic))",
4143	.test = alg_test_aead,
4144	.fips_allowed = 1,
4145	.suite = {
4146	.aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
4147	}
4148	}, {
4149	.alg = "authenc(hmac(sha256),cbc(des))",
4150	.generic_driver = "authenc(hmac-sha256-lib,cbc(des-generic))",
4151	.test = alg_test_aead,
4152	.suite = {
4153	.aead = __VECS(hmac_sha256_des_cbc_tv_temp)
4154	}
4155	}, {
4156	.alg = "authenc(hmac(sha256),cbc(des3_ede))",
4157	.generic_driver = "authenc(hmac-sha256-lib,cbc(des3_ede-generic))",
4158	.test = alg_test_aead,
4159	.suite = {
4160	.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
4161	}
4162	}, {
4163	.alg = "authenc(hmac(sha256),ctr(aes))",
4164	.test = alg_test_null,
4165	.fips_allowed = 1,
4166	}, {
4167	.alg = "authenc(hmac(sha256),cts(cbc(aes)))",
4168	.generic_driver = "authenc(hmac-sha256-lib,cts(cbc(aes-generic)))",
4169	.test = alg_test_aead,
4170	.suite = {
4171	.aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128)
4172	}
4173	}, {
4174	.alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4175	.test = alg_test_null,
4176	.fips_allowed = 1,
4177	}, {
4178	.alg = "authenc(hmac(sha384),cbc(des))",
4179	.generic_driver = "authenc(hmac-sha384-lib,cbc(des-generic))",
4180	.test = alg_test_aead,
4181	.suite = {
4182	.aead = __VECS(hmac_sha384_des_cbc_tv_temp)
4183	}
4184	}, {
4185	.alg = "authenc(hmac(sha384),cbc(des3_ede))",
4186	.generic_driver = "authenc(hmac-sha384-lib,cbc(des3_ede-generic))",
4187	.test = alg_test_aead,
4188	.suite = {
4189	.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
4190	}
4191	}, {
4192	.alg = "authenc(hmac(sha384),ctr(aes))",
4193	.test = alg_test_null,
4194	.fips_allowed = 1,
4195	}, {
4196	.alg = "authenc(hmac(sha384),cts(cbc(aes)))",
4197	.generic_driver = "authenc(hmac-sha384-lib,cts(cbc(aes-generic)))",
4198	.test = alg_test_aead,
4199	.suite = {
4200	.aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192)
4201	}
4202	}, {
4203	.alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4204	.test = alg_test_null,
4205	.fips_allowed = 1,
4206	}, {
4207	.alg = "authenc(hmac(sha512),cbc(aes))",
4208	.generic_driver = "authenc(hmac-sha512-lib,cbc(aes-generic))",
4209	.fips_allowed = 1,
4210	.test = alg_test_aead,
4211	.suite = {
4212	.aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
4213	}
4214	}, {
4215	.alg = "authenc(hmac(sha512),cbc(des))",
4216	.generic_driver = "authenc(hmac-sha512-lib,cbc(des-generic))",
4217	.test = alg_test_aead,
4218	.suite = {
4219	.aead = __VECS(hmac_sha512_des_cbc_tv_temp)
4220	}
4221	}, {
4222	.alg = "authenc(hmac(sha512),cbc(des3_ede))",
4223	.generic_driver = "authenc(hmac-sha512-lib,cbc(des3_ede-generic))",
4224	.test = alg_test_aead,
4225	.suite = {
4226	.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
4227	}
4228	}, {
4229	.alg = "authenc(hmac(sha512),ctr(aes))",
4230	.test = alg_test_null,
4231	.fips_allowed = 1,
4232	}, {
4233	.alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4234	.test = alg_test_null,
4235	.fips_allowed = 1,
4236	}, {
4237	.alg = "blake2b-160",
4238	.generic_driver = "blake2b-160-lib",
4239	.test = alg_test_hash,
4240	.fips_allowed = 0,
4241	.suite = {
4242	.hash = __VECS(blake2b_160_tv_template)
4243	}
4244	}, {
4245	.alg = "blake2b-256",
4246	.generic_driver = "blake2b-256-lib",
4247	.test = alg_test_hash,
4248	.fips_allowed = 0,
4249	.suite = {
4250	.hash = __VECS(blake2b_256_tv_template)
4251	}
4252	}, {
4253	.alg = "blake2b-384",
4254	.generic_driver = "blake2b-384-lib",
4255	.test = alg_test_hash,
4256	.fips_allowed = 0,
4257	.suite = {
4258	.hash = __VECS(blake2b_384_tv_template)
4259	}
4260	}, {
4261	.alg = "blake2b-512",
4262	.generic_driver = "blake2b-512-lib",
4263	.test = alg_test_hash,
4264	.fips_allowed = 0,
4265	.suite = {
4266	.hash = __VECS(blake2b_512_tv_template)
4267	}
4268	}, {
4269	.alg = "cbc(aes)",
4270	.test = alg_test_skcipher,
4271	.fips_allowed = 1,
4272	.suite = {
4273	.cipher = __VECS(aes_cbc_tv_template)
4274	},
4275	}, {
4276	.alg = "cbc(anubis)",
4277	.test = alg_test_skcipher,
4278	.suite = {
4279	.cipher = __VECS(anubis_cbc_tv_template)
4280	},
4281	}, {
4282	.alg = "cbc(aria)",
4283	.test = alg_test_skcipher,
4284	.suite = {
4285	.cipher = __VECS(aria_cbc_tv_template)
4286	},
4287	}, {
4288	.alg = "cbc(blowfish)",
4289	.test = alg_test_skcipher,
4290	.suite = {
4291	.cipher = __VECS(bf_cbc_tv_template)
4292	},
4293	}, {
4294	.alg = "cbc(camellia)",
4295	.test = alg_test_skcipher,
4296	.suite = {
4297	.cipher = __VECS(camellia_cbc_tv_template)
4298	},
4299	}, {
4300	.alg = "cbc(cast5)",
4301	.test = alg_test_skcipher,
4302	.suite = {
4303	.cipher = __VECS(cast5_cbc_tv_template)
4304	},
4305	}, {
4306	.alg = "cbc(cast6)",
4307	.test = alg_test_skcipher,
4308	.suite = {
4309	.cipher = __VECS(cast6_cbc_tv_template)
4310	},
4311	}, {
4312	.alg = "cbc(des)",
4313	.test = alg_test_skcipher,
4314	.suite = {
4315	.cipher = __VECS(des_cbc_tv_template)
4316	},
4317	}, {
4318	.alg = "cbc(des3_ede)",
4319	.test = alg_test_skcipher,
4320	.suite = {
4321	.cipher = __VECS(des3_ede_cbc_tv_template)
4322	},
4323	}, {
4324	/* Same as cbc(aes) except the key is stored in
4325	* hardware secure memory which we reference by index
4326	*/
4327	.alg = "cbc(paes)",
4328	.test = alg_test_null,
4329	.fips_allowed = 1,
4330	}, {
4331	/* Same as cbc(sm4) except the key is stored in
4332	* hardware secure memory which we reference by index
4333	*/
4334	.alg = "cbc(psm4)",
4335	.test = alg_test_null,
4336	}, {
4337	.alg = "cbc(serpent)",
4338	.test = alg_test_skcipher,
4339	.suite = {
4340	.cipher = __VECS(serpent_cbc_tv_template)
4341	},
4342	}, {
4343	.alg = "cbc(sm4)",
4344	.test = alg_test_skcipher,
4345	.suite = {
4346	.cipher = __VECS(sm4_cbc_tv_template)
4347	}
4348	}, {
4349	.alg = "cbc(twofish)",
4350	.test = alg_test_skcipher,
4351	.suite = {
4352	.cipher = __VECS(tf_cbc_tv_template)
4353	},
4354	}, {
4355	#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4356	.alg = "cbc-paes-s390",
4357	.fips_allowed = 1,
4358	.test = alg_test_skcipher,
4359	.suite = {
4360	.cipher = __VECS(aes_cbc_tv_template)
4361	}
4362	}, {
4363	#endif
4364	.alg = "cbcmac(aes)",
4365	.test = alg_test_hash,
4366	.suite = {
4367	.hash = __VECS(aes_cbcmac_tv_template)
4368	}
4369	}, {
4370	.alg = "cbcmac(sm4)",
4371	.test = alg_test_hash,
4372	.suite = {
4373	.hash = __VECS(sm4_cbcmac_tv_template)
4374	}
4375	}, {
4376	.alg = "ccm(aes)",
4377	.generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
4378	.test = alg_test_aead,
4379	.fips_allowed = 1,
4380	.suite = {
4381	.aead = {
4382	____VECS(aes_ccm_tv_template),
4383	.einval_allowed = 1,
4384	}
4385	}
4386	}, {
4387	.alg = "ccm(sm4)",
4388	.generic_driver = "ccm_base(ctr(sm4-generic),cbcmac(sm4-generic))",
4389	.test = alg_test_aead,
4390	.suite = {
4391	.aead = {
4392	____VECS(sm4_ccm_tv_template),
4393	.einval_allowed = 1,
4394	}
4395	}
4396	}, {
4397	.alg = "chacha20",
4398	.generic_driver = "chacha20-lib",
4399	.test = alg_test_skcipher,
4400	.suite = {
4401	.cipher = __VECS(chacha20_tv_template)
4402	},
4403	}, {
4404	.alg = "cmac(aes)",
4405	.fips_allowed = 1,
4406	.test = alg_test_hash,
4407	.suite = {
4408	.hash = __VECS(aes_cmac128_tv_template)
4409	}
4410	}, {
4411	.alg = "cmac(camellia)",
4412	.test = alg_test_hash,
4413	.suite = {
4414	.hash = __VECS(camellia_cmac128_tv_template)
4415	}
4416	}, {
4417	.alg = "cmac(des3_ede)",
4418	.test = alg_test_hash,
4419	.suite = {
4420	.hash = __VECS(des3_ede_cmac64_tv_template)
4421	}
4422	}, {
4423	.alg = "cmac(sm4)",
4424	.test = alg_test_hash,
4425	.suite = {
4426	.hash = __VECS(sm4_cmac128_tv_template)
4427	}
4428	}, {
4429	.alg = "crc32",
4430	.generic_driver = "crc32-lib",
4431	.test = alg_test_hash,
4432	.fips_allowed = 1,
4433	.suite = {
4434	.hash = __VECS(crc32_tv_template)
4435	}
4436	}, {
4437	.alg = "crc32c",
4438	.generic_driver = "crc32c-lib",
4439	.test = alg_test_hash,
4440	.fips_allowed = 1,
4441	.suite = {
4442	.hash = __VECS(crc32c_tv_template)
4443	}
4444	}, {
4445	.alg = "ctr(aes)",
4446	.test = alg_test_skcipher,
4447	.fips_allowed = 1,
4448	.suite = {
4449	.cipher = __VECS(aes_ctr_tv_template)
4450	}
4451	}, {
4452	.alg = "ctr(aria)",
4453	.test = alg_test_skcipher,
4454	.suite = {
4455	.cipher = __VECS(aria_ctr_tv_template)
4456	}
4457	}, {
4458	.alg = "ctr(blowfish)",
4459	.test = alg_test_skcipher,
4460	.suite = {
4461	.cipher = __VECS(bf_ctr_tv_template)
4462	}
4463	}, {
4464	.alg = "ctr(camellia)",
4465	.test = alg_test_skcipher,
4466	.suite = {
4467	.cipher = __VECS(camellia_ctr_tv_template)
4468	}
4469	}, {
4470	.alg = "ctr(cast5)",
4471	.test = alg_test_skcipher,
4472	.suite = {
4473	.cipher = __VECS(cast5_ctr_tv_template)
4474	}
4475	}, {
4476	.alg = "ctr(cast6)",
4477	.test = alg_test_skcipher,
4478	.suite = {
4479	.cipher = __VECS(cast6_ctr_tv_template)
4480	}
4481	}, {
4482	.alg = "ctr(des)",
4483	.test = alg_test_skcipher,
4484	.suite = {
4485	.cipher = __VECS(des_ctr_tv_template)
4486	}
4487	}, {
4488	.alg = "ctr(des3_ede)",
4489	.test = alg_test_skcipher,
4490	.suite = {
4491	.cipher = __VECS(des3_ede_ctr_tv_template)
4492	}
4493	}, {
4494	/* Same as ctr(aes) except the key is stored in
4495	* hardware secure memory which we reference by index
4496	*/
4497	.alg = "ctr(paes)",
4498	.test = alg_test_null,
4499	.fips_allowed = 1,
4500	}, {
4501
4502	/* Same as ctr(sm4) except the key is stored in
4503	* hardware secure memory which we reference by index
4504	*/
4505	.alg = "ctr(psm4)",
4506	.test = alg_test_null,
4507	}, {
4508	.alg = "ctr(serpent)",
4509	.test = alg_test_skcipher,
4510	.suite = {
4511	.cipher = __VECS(serpent_ctr_tv_template)
4512	}
4513	}, {
4514	.alg = "ctr(sm4)",
4515	.test = alg_test_skcipher,
4516	.suite = {
4517	.cipher = __VECS(sm4_ctr_tv_template)
4518	}
4519	}, {
4520	.alg = "ctr(twofish)",
4521	.test = alg_test_skcipher,
4522	.suite = {
4523	.cipher = __VECS(tf_ctr_tv_template)
4524	}
4525	}, {
4526	#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4527	.alg = "ctr-paes-s390",
4528	.fips_allowed = 1,
4529	.test = alg_test_skcipher,
4530	.suite = {
4531	.cipher = __VECS(aes_ctr_tv_template)
4532	}
4533	}, {
4534	#endif
4535	.alg = "cts(cbc(aes))",
4536	.test = alg_test_skcipher,
4537	.fips_allowed = 1,
4538	.suite = {
4539	.cipher = __VECS(cts_mode_tv_template)
4540	}
4541	}, {
4542	/* Same as cts(cbc((aes)) except the key is stored in
4543	* hardware secure memory which we reference by index
4544	*/
4545	.alg = "cts(cbc(paes))",
4546	.test = alg_test_null,
4547	.fips_allowed = 1,
4548	}, {
4549	.alg = "cts(cbc(sm4))",
4550	.test = alg_test_skcipher,
4551	.suite = {
4552	.cipher = __VECS(sm4_cts_tv_template)
4553	}
4554	}, {
4555	.alg = "deflate",
4556	.test = alg_test_comp,
4557	.fips_allowed = 1,
4558	.suite = {
4559	.comp = {
4560	.comp = __VECS(deflate_comp_tv_template),
4561	.decomp = __VECS(deflate_decomp_tv_template)
4562	}
4563	}
4564	}, {
4565	.alg = "deflate-iaa",
4566	.test = alg_test_comp,
4567	.fips_allowed = 1,
4568	.suite = {
4569	.comp = {
4570	.comp = __VECS(deflate_comp_tv_template),
4571	.decomp = __VECS(deflate_decomp_tv_template)
4572	}
4573	}
4574	}, {
4575	.alg = "dh",
4576	.test = alg_test_kpp,
4577	.suite = {
4578	.kpp = __VECS(dh_tv_template)
4579	}
4580	}, {
4581	.alg = "digest_null",
4582	.test = alg_test_null,
4583	}, {
4584	.alg = "drbg_nopr_ctr_aes128",
4585	.test = alg_test_drbg,
4586	.fips_allowed = 1,
4587	.suite = {
4588	.drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
4589	}
4590	}, {
4591	.alg = "drbg_nopr_ctr_aes192",
4592	.test = alg_test_drbg,
4593	.fips_allowed = 1,
4594	.suite = {
4595	.drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4596	}
4597	}, {
4598	.alg = "drbg_nopr_ctr_aes256",
4599	.test = alg_test_drbg,
4600	.fips_allowed = 1,
4601	.suite = {
4602	.drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4603	}
4604	}, {
4605	.alg = "drbg_nopr_hmac_sha256",
4606	.test = alg_test_drbg,
4607	.fips_allowed = 1,
4608	.suite = {
4609	.drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4610	}
4611	}, {
4612	/*
4613	* There is no need to specifically test the DRBG with every
4614	* backend cipher -- covered by drbg_nopr_hmac_sha512 test
4615	*/
4616	.alg = "drbg_nopr_hmac_sha384",
4617	.test = alg_test_null,
4618	.fips_allowed = 1
4619	}, {
4620	.alg = "drbg_nopr_hmac_sha512",
4621	.test = alg_test_drbg,
4622	.fips_allowed = 1,
4623	.suite = {
4624	.drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
4625	}
4626	}, {
4627	.alg = "drbg_nopr_sha256",
4628	.test = alg_test_drbg,
4629	.fips_allowed = 1,
4630	.suite = {
4631	.drbg = __VECS(drbg_nopr_sha256_tv_template)
4632	}
4633	}, {
4634	/* covered by drbg_nopr_sha256 test */
4635	.alg = "drbg_nopr_sha384",
4636	.test = alg_test_null,
4637	.fips_allowed = 1
4638	}, {
4639	.alg = "drbg_nopr_sha512",
4640	.fips_allowed = 1,
4641	.test = alg_test_null,
4642	}, {
4643	.alg = "drbg_pr_ctr_aes128",
4644	.test = alg_test_drbg,
4645	.fips_allowed = 1,
4646	.suite = {
4647	.drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4648	}
4649	}, {
4650	/* covered by drbg_pr_ctr_aes128 test */
4651	.alg = "drbg_pr_ctr_aes192",
4652	.fips_allowed = 1,
4653	.test = alg_test_null,
4654	}, {
4655	.alg = "drbg_pr_ctr_aes256",
4656	.fips_allowed = 1,
4657	.test = alg_test_null,
4658	}, {
4659	.alg = "drbg_pr_hmac_sha256",
4660	.test = alg_test_drbg,
4661	.fips_allowed = 1,
4662	.suite = {
4663	.drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
4664	}
4665	}, {
4666	/* covered by drbg_pr_hmac_sha256 test */
4667	.alg = "drbg_pr_hmac_sha384",
4668	.test = alg_test_null,
4669	.fips_allowed = 1
4670	}, {
4671	.alg = "drbg_pr_hmac_sha512",
4672	.test = alg_test_null,
4673	.fips_allowed = 1,
4674	}, {
4675	.alg = "drbg_pr_sha256",
4676	.test = alg_test_drbg,
4677	.fips_allowed = 1,
4678	.suite = {
4679	.drbg = __VECS(drbg_pr_sha256_tv_template)
4680	}
4681	}, {
4682	/* covered by drbg_pr_sha256 test */
4683	.alg = "drbg_pr_sha384",
4684	.test = alg_test_null,
4685	.fips_allowed = 1
4686	}, {
4687	.alg = "drbg_pr_sha512",
4688	.fips_allowed = 1,
4689	.test = alg_test_null,
4690	}, {
4691	.alg = "ecb(aes)",
4692	.test = alg_test_skcipher,
4693	.fips_allowed = 1,
4694	.suite = {
4695	.cipher = __VECS(aes_tv_template)
4696	}
4697	}, {
4698	.alg = "ecb(anubis)",
4699	.test = alg_test_skcipher,
4700	.suite = {
4701	.cipher = __VECS(anubis_tv_template)
4702	}
4703	}, {
4704	.alg = "ecb(arc4)",
4705	.generic_driver = "arc4-generic",
4706	.test = alg_test_skcipher,
4707	.suite = {
4708	.cipher = __VECS(arc4_tv_template)
4709	}
4710	}, {
4711	.alg = "ecb(aria)",
4712	.test = alg_test_skcipher,
4713	.suite = {
4714	.cipher = __VECS(aria_tv_template)
4715	}
4716	}, {
4717	.alg = "ecb(blowfish)",
4718	.test = alg_test_skcipher,
4719	.suite = {
4720	.cipher = __VECS(bf_tv_template)
4721	}
4722	}, {
4723	.alg = "ecb(camellia)",
4724	.test = alg_test_skcipher,
4725	.suite = {
4726	.cipher = __VECS(camellia_tv_template)
4727	}
4728	}, {
4729	.alg = "ecb(cast5)",
4730	.test = alg_test_skcipher,
4731	.suite = {
4732	.cipher = __VECS(cast5_tv_template)
4733	}
4734	}, {
4735	.alg = "ecb(cast6)",
4736	.test = alg_test_skcipher,
4737	.suite = {
4738	.cipher = __VECS(cast6_tv_template)
4739	}
4740	}, {
4741	.alg = "ecb(cipher_null)",
4742	.test = alg_test_null,
4743	.fips_allowed = 1,
4744	}, {
4745	.alg = "ecb(des)",
4746	.test = alg_test_skcipher,
4747	.suite = {
4748	.cipher = __VECS(des_tv_template)
4749	}
4750	}, {
4751	.alg = "ecb(des3_ede)",
4752	.test = alg_test_skcipher,
4753	.suite = {
4754	.cipher = __VECS(des3_ede_tv_template)
4755	}
4756	}, {
4757	.alg = "ecb(fcrypt)",
4758	.test = alg_test_skcipher,
4759	.suite = {
4760	.cipher = {
4761	.vecs = fcrypt_pcbc_tv_template,
4762	.count = 1
4763	}
4764	}
4765	}, {
4766	.alg = "ecb(khazad)",
4767	.test = alg_test_skcipher,
4768	.suite = {
4769	.cipher = __VECS(khazad_tv_template)
4770	}
4771	}, {
4772	/* Same as ecb(aes) except the key is stored in
4773	* hardware secure memory which we reference by index
4774	*/
4775	.alg = "ecb(paes)",
4776	.test = alg_test_null,
4777	.fips_allowed = 1,
4778	}, {
4779	.alg = "ecb(seed)",
4780	.test = alg_test_skcipher,
4781	.suite = {
4782	.cipher = __VECS(seed_tv_template)
4783	}
4784	}, {
4785	.alg = "ecb(serpent)",
4786	.test = alg_test_skcipher,
4787	.suite = {
4788	.cipher = __VECS(serpent_tv_template)
4789	}
4790	}, {
4791	.alg = "ecb(sm4)",
4792	.test = alg_test_skcipher,
4793	.suite = {
4794	.cipher = __VECS(sm4_tv_template)
4795	}
4796	}, {
4797	.alg = "ecb(tea)",
4798	.test = alg_test_skcipher,
4799	.suite = {
4800	.cipher = __VECS(tea_tv_template)
4801	}
4802	}, {
4803	.alg = "ecb(twofish)",
4804	.test = alg_test_skcipher,
4805	.suite = {
4806	.cipher = __VECS(tf_tv_template)
4807	}
4808	}, {
4809	.alg = "ecb(xeta)",
4810	.test = alg_test_skcipher,
4811	.suite = {
4812	.cipher = __VECS(xeta_tv_template)
4813	}
4814	}, {
4815	.alg = "ecb(xtea)",
4816	.test = alg_test_skcipher,
4817	.suite = {
4818	.cipher = __VECS(xtea_tv_template)
4819	}
4820	}, {
4821	#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4822	.alg = "ecb-paes-s390",
4823	.fips_allowed = 1,
4824	.test = alg_test_skcipher,
4825	.suite = {
4826	.cipher = __VECS(aes_tv_template)
4827	}
4828	}, {
4829	#endif
4830	.alg = "ecdh-nist-p192",
4831	.test = alg_test_kpp,
4832	.suite = {
4833	.kpp = __VECS(ecdh_p192_tv_template)
4834	}
4835	}, {
4836	.alg = "ecdh-nist-p256",
4837	.test = alg_test_kpp,
4838	.fips_allowed = 1,
4839	.suite = {
4840	.kpp = __VECS(ecdh_p256_tv_template)
4841	}
4842	}, {
4843	.alg = "ecdh-nist-p384",
4844	.test = alg_test_kpp,
4845	.fips_allowed = 1,
4846	.suite = {
4847	.kpp = __VECS(ecdh_p384_tv_template)
4848	}
4849	}, {
4850	.alg = "ecdsa-nist-p192",
4851	.test = alg_test_sig,
4852	.suite = {
4853	.sig = __VECS(ecdsa_nist_p192_tv_template)
4854	}
4855	}, {
4856	.alg = "ecdsa-nist-p256",
4857	.test = alg_test_sig,
4858	.fips_allowed = 1,
4859	.suite = {
4860	.sig = __VECS(ecdsa_nist_p256_tv_template)
4861	}
4862	}, {
4863	.alg = "ecdsa-nist-p384",
4864	.test = alg_test_sig,
4865	.fips_allowed = 1,
4866	.suite = {
4867	.sig = __VECS(ecdsa_nist_p384_tv_template)
4868	}
4869	}, {
4870	.alg = "ecdsa-nist-p521",
4871	.test = alg_test_sig,
4872	.fips_allowed = 1,
4873	.suite = {
4874	.sig = __VECS(ecdsa_nist_p521_tv_template)
4875	}
4876	}, {
4877	.alg = "ecrdsa",
4878	.test = alg_test_sig,
4879	.suite = {
4880	.sig = __VECS(ecrdsa_tv_template)
4881	}
4882	}, {
4883	.alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
4884	.generic_driver = "essiv(authenc(hmac-sha256-lib,cbc(aes-generic)),sha256-lib)",
4885	.test = alg_test_aead,
4886	.fips_allowed = 1,
4887	.suite = {
4888	.aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
4889	}
4890	}, {
4891	.alg = "essiv(cbc(aes),sha256)",
4892	.generic_driver = "essiv(cbc(aes-generic),sha256-lib)",
4893	.test = alg_test_skcipher,
4894	.fips_allowed = 1,
4895	.suite = {
4896	.cipher = __VECS(essiv_aes_cbc_tv_template)
4897	}
4898	}, {
4899	#if IS_ENABLED(CONFIG_CRYPTO_DH_RFC7919_GROUPS)
4900	.alg = "ffdhe2048(dh)",
4901	.test = alg_test_kpp,
4902	.fips_allowed = 1,
4903	.suite = {
4904	.kpp = __VECS(ffdhe2048_dh_tv_template)
4905	}
4906	}, {
4907	.alg = "ffdhe3072(dh)",
4908	.test = alg_test_kpp,
4909	.fips_allowed = 1,
4910	.suite = {
4911	.kpp = __VECS(ffdhe3072_dh_tv_template)
4912	}
4913	}, {
4914	.alg = "ffdhe4096(dh)",
4915	.test = alg_test_kpp,
4916	.fips_allowed = 1,
4917	.suite = {
4918	.kpp = __VECS(ffdhe4096_dh_tv_template)
4919	}
4920	}, {
4921	.alg = "ffdhe6144(dh)",
4922	.test = alg_test_kpp,
4923	.fips_allowed = 1,
4924	.suite = {
4925	.kpp = __VECS(ffdhe6144_dh_tv_template)
4926	}
4927	}, {
4928	.alg = "ffdhe8192(dh)",
4929	.test = alg_test_kpp,
4930	.fips_allowed = 1,
4931	.suite = {
4932	.kpp = __VECS(ffdhe8192_dh_tv_template)
4933	}
4934	}, {
4935	#endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */
4936	.alg = "gcm(aes)",
4937	.generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
4938	.test = alg_test_aead,
4939	.fips_allowed = 1,
4940	.suite = {
4941	.aead = __VECS(aes_gcm_tv_template)
4942	}
4943	}, {
4944	.alg = "gcm(aria)",
4945	.generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)",
4946	.test = alg_test_aead,
4947	.suite = {
4948	.aead = __VECS(aria_gcm_tv_template)
4949	}
4950	}, {
4951	.alg = "gcm(sm4)",
4952	.generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)",
4953	.test = alg_test_aead,
4954	.suite = {
4955	.aead = __VECS(sm4_gcm_tv_template)
4956	}
4957	}, {
4958	.alg = "ghash",
4959	.test = alg_test_hash,
4960	.suite = {
4961	.hash = __VECS(ghash_tv_template)
4962	}
4963	}, {
4964	.alg = "hctr2(aes)",
4965	.generic_driver = "hctr2_base(xctr(aes-generic),polyval-lib)",
4966	.test = alg_test_skcipher,
4967	.suite = {
4968	.cipher = __VECS(aes_hctr2_tv_template)
4969	}
4970	}, {
4971	.alg = "hmac(md5)",
4972	.generic_driver = "hmac-md5-lib",
4973	.test = alg_test_hash,
4974	.suite = {
4975	.hash = __VECS(hmac_md5_tv_template)
4976	}
4977	}, {
4978	.alg = "hmac(rmd160)",
4979	.test = alg_test_hash,
4980	.suite = {
4981	.hash = __VECS(hmac_rmd160_tv_template)
4982	}
4983	}, {
4984	.alg = "hmac(sha1)",
4985	.generic_driver = "hmac-sha1-lib",
4986	.test = alg_test_hash,
4987	.fips_allowed = 1,
4988	.suite = {
4989	.hash = __VECS(hmac_sha1_tv_template)
4990	}
4991	}, {
4992	.alg = "hmac(sha224)",
4993	.generic_driver = "hmac-sha224-lib",
4994	.test = alg_test_hash,
4995	.fips_allowed = 1,
4996	.suite = {
4997	.hash = __VECS(hmac_sha224_tv_template)
4998	}
4999	}, {
5000	.alg = "hmac(sha256)",
5001	.generic_driver = "hmac-sha256-lib",
5002	.test = alg_test_hash,
5003	.fips_allowed = 1,
5004	.suite = {
5005	.hash = __VECS(hmac_sha256_tv_template)
5006	}
5007	}, {
5008	.alg = "hmac(sha3-224)",
5009	.generic_driver = "hmac(sha3-224-lib)",
5010	.test = alg_test_hash,
5011	.fips_allowed = 1,
5012	.suite = {
5013	.hash = __VECS(hmac_sha3_224_tv_template)
5014	}
5015	}, {
5016	.alg = "hmac(sha3-256)",
5017	.generic_driver = "hmac(sha3-256-lib)",
5018	.test = alg_test_hash,
5019	.fips_allowed = 1,
5020	.suite = {
5021	.hash = __VECS(hmac_sha3_256_tv_template)
5022	}
5023	}, {
5024	.alg = "hmac(sha3-384)",
5025	.generic_driver = "hmac(sha3-384-lib)",
5026	.test = alg_test_hash,
5027	.fips_allowed = 1,
5028	.suite = {
5029	.hash = __VECS(hmac_sha3_384_tv_template)
5030	}
5031	}, {
5032	.alg = "hmac(sha3-512)",
5033	.generic_driver = "hmac(sha3-512-lib)",
5034	.test = alg_test_hash,
5035	.fips_allowed = 1,
5036	.suite = {
5037	.hash = __VECS(hmac_sha3_512_tv_template)
5038	}
5039	}, {
5040	.alg = "hmac(sha384)",
5041	.generic_driver = "hmac-sha384-lib",
5042	.test = alg_test_hash,
5043	.fips_allowed = 1,
5044	.suite = {
5045	.hash = __VECS(hmac_sha384_tv_template)
5046	}
5047	}, {
5048	.alg = "hmac(sha512)",
5049	.generic_driver = "hmac-sha512-lib",
5050	.test = alg_test_hash,
5051	.fips_allowed = 1,
5052	.suite = {
5053	.hash = __VECS(hmac_sha512_tv_template)
5054	}
5055	}, {
5056	.alg = "hmac(sm3)",
5057	.test = alg_test_hash,
5058	.suite = {
5059	.hash = __VECS(hmac_sm3_tv_template)
5060	}
5061	}, {
5062	.alg = "hmac(streebog256)",
5063	.test = alg_test_hash,
5064	.suite = {
5065	.hash = __VECS(hmac_streebog256_tv_template)
5066	}
5067	}, {
5068	.alg = "hmac(streebog512)",
5069	.test = alg_test_hash,
5070	.suite = {
5071	.hash = __VECS(hmac_streebog512_tv_template)
5072	}
5073	}, {
5074	.alg = "jitterentropy_rng",
5075	.fips_allowed = 1,
5076	.test = alg_test_null,
5077	}, {
5078	.alg = "krb5enc(cmac(camellia),cts(cbc(camellia)))",
5079	.test = alg_test_aead,
5080	.suite.aead = __VECS(krb5_test_camellia_cts_cmac)
5081	}, {
5082	.alg = "lrw(aes)",
5083	.generic_driver = "lrw(ecb(aes-generic))",
5084	.test = alg_test_skcipher,
5085	.suite = {
5086	.cipher = __VECS(aes_lrw_tv_template)
5087	}
5088	}, {
5089	.alg = "lrw(camellia)",
5090	.generic_driver = "lrw(ecb(camellia-generic))",
5091	.test = alg_test_skcipher,
5092	.suite = {
5093	.cipher = __VECS(camellia_lrw_tv_template)
5094	}
5095	}, {
5096	.alg = "lrw(cast6)",
5097	.generic_driver = "lrw(ecb(cast6-generic))",
5098	.test = alg_test_skcipher,
5099	.suite = {
5100	.cipher = __VECS(cast6_lrw_tv_template)
5101	}
5102	}, {
5103	.alg = "lrw(serpent)",
5104	.generic_driver = "lrw(ecb(serpent-generic))",
5105	.test = alg_test_skcipher,
5106	.suite = {
5107	.cipher = __VECS(serpent_lrw_tv_template)
5108	}
5109	}, {
5110	.alg = "lrw(twofish)",
5111	.generic_driver = "lrw(ecb(twofish-generic))",
5112	.test = alg_test_skcipher,
5113	.suite = {
5114	.cipher = __VECS(tf_lrw_tv_template)
5115	}
5116	}, {
5117	.alg = "lz4",
5118	.test = alg_test_comp,
5119	.fips_allowed = 1,
5120	.suite = {
5121	.comp = {
5122	.comp = __VECS(lz4_comp_tv_template),
5123	.decomp = __VECS(lz4_decomp_tv_template)
5124	}
5125	}
5126	}, {
5127	.alg = "lz4hc",
5128	.test = alg_test_comp,
5129	.fips_allowed = 1,
5130	.suite = {
5131	.comp = {
5132	.comp = __VECS(lz4hc_comp_tv_template),
5133	.decomp = __VECS(lz4hc_decomp_tv_template)
5134	}
5135	}
5136	}, {
5137	.alg = "lzo",
5138	.test = alg_test_comp,
5139	.fips_allowed = 1,
5140	.suite = {
5141	.comp = {
5142	.comp = __VECS(lzo_comp_tv_template),
5143	.decomp = __VECS(lzo_decomp_tv_template)
5144	}
5145	}
5146	}, {
5147	.alg = "lzo-rle",
5148	.test = alg_test_comp,
5149	.fips_allowed = 1,
5150	.suite = {
5151	.comp = {
5152	.comp = __VECS(lzorle_comp_tv_template),
5153	.decomp = __VECS(lzorle_decomp_tv_template)
5154	}
5155	}
5156	}, {
5157	.alg = "md4",
5158	.test = alg_test_hash,
5159	.suite = {
5160	.hash = __VECS(md4_tv_template)
5161	}
5162	}, {
5163	.alg = "md5",
5164	.generic_driver = "md5-lib",
5165	.test = alg_test_hash,
5166	.suite = {
5167	.hash = __VECS(md5_tv_template)
5168	}
5169	}, {
5170	.alg = "michael_mic",
5171	.test = alg_test_hash,
5172	.suite = {
5173	.hash = __VECS(michael_mic_tv_template)
5174	}
5175	}, {
5176	.alg = "nhpoly1305",
5177	.test = alg_test_hash,
5178	.suite = {
5179	.hash = __VECS(nhpoly1305_tv_template)
5180	}
5181	}, {
5182	.alg = "p1363(ecdsa-nist-p192)",
5183	.test = alg_test_null,
5184	}, {
5185	.alg = "p1363(ecdsa-nist-p256)",
5186	.test = alg_test_sig,
5187	.fips_allowed = 1,
5188	.suite = {
5189	.sig = __VECS(p1363_ecdsa_nist_p256_tv_template)
5190	}
5191	}, {
5192	.alg = "p1363(ecdsa-nist-p384)",
5193	.test = alg_test_null,
5194	.fips_allowed = 1,
5195	}, {
5196	.alg = "p1363(ecdsa-nist-p521)",
5197	.test = alg_test_null,
5198	.fips_allowed = 1,
5199	}, {
5200	.alg = "pcbc(fcrypt)",
5201	.test = alg_test_skcipher,
5202	.suite = {
5203	.cipher = __VECS(fcrypt_pcbc_tv_template)
5204	}
5205	}, {
5206	#if IS_ENABLED(CONFIG_CRYPTO_PHMAC_S390)
5207	.alg = "phmac(sha224)",
5208	.test = alg_test_hash,
5209	.fips_allowed = 1,
5210	.suite = {
5211	.hash = __VECS(hmac_sha224_tv_template)
5212	}
5213	}, {
5214	.alg = "phmac(sha256)",
5215	.test = alg_test_hash,
5216	.fips_allowed = 1,
5217	.suite = {
5218	.hash = __VECS(hmac_sha256_tv_template)
5219	}
5220	}, {
5221	.alg = "phmac(sha384)",
5222	.test = alg_test_hash,
5223	.fips_allowed = 1,
5224	.suite = {
5225	.hash = __VECS(hmac_sha384_tv_template)
5226	}
5227	}, {
5228	.alg = "phmac(sha512)",
5229	.test = alg_test_hash,
5230	.fips_allowed = 1,
5231	.suite = {
5232	.hash = __VECS(hmac_sha512_tv_template)
5233	}
5234	}, {
5235	#endif
5236	.alg = "pkcs1(rsa,none)",
5237	.test = alg_test_sig,
5238	.suite = {
5239	.sig = __VECS(pkcs1_rsa_none_tv_template)
5240	}
5241	}, {
5242	.alg = "pkcs1(rsa,sha224)",
5243	.test = alg_test_null,
5244	.fips_allowed = 1,
5245	}, {
5246	.alg = "pkcs1(rsa,sha256)",
5247	.test = alg_test_sig,
5248	.fips_allowed = 1,
5249	.suite = {
5250	.sig = __VECS(pkcs1_rsa_tv_template)
5251	}
5252	}, {
5253	.alg = "pkcs1(rsa,sha3-256)",
5254	.test = alg_test_null,
5255	.fips_allowed = 1,
5256	}, {
5257	.alg = "pkcs1(rsa,sha3-384)",
5258	.test = alg_test_null,
5259	.fips_allowed = 1,
5260	}, {
5261	.alg = "pkcs1(rsa,sha3-512)",
5262	.test = alg_test_null,
5263	.fips_allowed = 1,
5264	}, {
5265	.alg = "pkcs1(rsa,sha384)",
5266	.test = alg_test_null,
5267	.fips_allowed = 1,
5268	}, {
5269	.alg = "pkcs1(rsa,sha512)",
5270	.test = alg_test_null,
5271	.fips_allowed = 1,
5272	}, {
5273	.alg = "pkcs1pad(rsa)",
5274	.test = alg_test_null,
5275	.fips_allowed = 1,
5276	}, {
5277	.alg = "rfc3686(ctr(aes))",
5278	.test = alg_test_skcipher,
5279	.fips_allowed = 1,
5280	.suite = {
5281	.cipher = __VECS(aes_ctr_rfc3686_tv_template)
5282	}
5283	}, {
5284	.alg = "rfc3686(ctr(sm4))",
5285	.test = alg_test_skcipher,
5286	.suite = {
5287	.cipher = __VECS(sm4_ctr_rfc3686_tv_template)
5288	}
5289	}, {
5290	.alg = "rfc4106(gcm(aes))",
5291	.generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
5292	.test = alg_test_aead,
5293	.fips_allowed = 1,
5294	.suite = {
5295	.aead = {
5296	____VECS(aes_gcm_rfc4106_tv_template),
5297	.einval_allowed = 1,
5298	.aad_iv = 1,
5299	}
5300	}
5301	}, {
5302	.alg = "rfc4309(ccm(aes))",
5303	.generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
5304	.test = alg_test_aead,
5305	.fips_allowed = 1,
5306	.suite = {
5307	.aead = {
5308	____VECS(aes_ccm_rfc4309_tv_template),
5309	.einval_allowed = 1,
5310	.aad_iv = 1,
5311	}
5312	}
5313	}, {
5314	.alg = "rfc4543(gcm(aes))",
5315	.generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
5316	.test = alg_test_aead,
5317	.suite = {
5318	.aead = {
5319	____VECS(aes_gcm_rfc4543_tv_template),
5320	.einval_allowed = 1,
5321	.aad_iv = 1,
5322	}
5323	}
5324	}, {
5325	.alg = "rfc7539(chacha20,poly1305)",
5326	.generic_driver = "rfc7539(chacha20-lib,poly1305-generic)",
5327	.test = alg_test_aead,
5328	.suite = {
5329	.aead = __VECS(rfc7539_tv_template)
5330	}
5331	}, {
5332	.alg = "rfc7539esp(chacha20,poly1305)",
5333	.generic_driver = "rfc7539esp(chacha20-lib,poly1305-generic)",
5334	.test = alg_test_aead,
5335	.suite = {
5336	.aead = {
5337	____VECS(rfc7539esp_tv_template),
5338	.einval_allowed = 1,
5339	.aad_iv = 1,
5340	}
5341	}
5342	}, {
5343	.alg = "rmd160",
5344	.test = alg_test_hash,
5345	.suite = {
5346	.hash = __VECS(rmd160_tv_template)
5347	}
5348	}, {
5349	.alg = "rsa",
5350	.test = alg_test_akcipher,
5351	.fips_allowed = 1,
5352	.suite = {
5353	.akcipher = __VECS(rsa_tv_template)
5354	}
5355	}, {
5356	.alg = "sha1",
5357	.generic_driver = "sha1-lib",
5358	.test = alg_test_hash,
5359	.fips_allowed = 1,
5360	.suite = {
5361	.hash = __VECS(sha1_tv_template)
5362	}
5363	}, {
5364	.alg = "sha224",
5365	.generic_driver = "sha224-lib",
5366	.test = alg_test_hash,
5367	.fips_allowed = 1,
5368	.suite = {
5369	.hash = __VECS(sha224_tv_template)
5370	}
5371	}, {
5372	.alg = "sha256",
5373	.generic_driver = "sha256-lib",
5374	.test = alg_test_hash,
5375	.fips_allowed = 1,
5376	.suite = {
5377	.hash = __VECS(sha256_tv_template)
5378	}
5379	}, {
5380	.alg = "sha3-224",
5381	.generic_driver = "sha3-224-lib",
5382	.test = alg_test_hash,
5383	.fips_allowed = 1,
5384	.suite = {
5385	.hash = __VECS(sha3_224_tv_template)
5386	}
5387	}, {
5388	.alg = "sha3-256",
5389	.generic_driver = "sha3-256-lib",
5390	.test = alg_test_hash,
5391	.fips_allowed = 1,
5392	.suite = {
5393	.hash = __VECS(sha3_256_tv_template)
5394	}
5395	}, {
5396	.alg = "sha3-384",
5397	.generic_driver = "sha3-384-lib",
5398	.test = alg_test_hash,
5399	.fips_allowed = 1,
5400	.suite = {
5401	.hash = __VECS(sha3_384_tv_template)
5402	}
5403	}, {
5404	.alg = "sha3-512",
5405	.generic_driver = "sha3-512-lib",
5406	.test = alg_test_hash,
5407	.fips_allowed = 1,
5408	.suite = {
5409	.hash = __VECS(sha3_512_tv_template)
5410	}
5411	}, {
5412	.alg = "sha384",
5413	.generic_driver = "sha384-lib",
5414	.test = alg_test_hash,
5415	.fips_allowed = 1,
5416	.suite = {
5417	.hash = __VECS(sha384_tv_template)
5418	}
5419	}, {
5420	.alg = "sha512",
5421	.generic_driver = "sha512-lib",
5422	.test = alg_test_hash,
5423	.fips_allowed = 1,
5424	.suite = {
5425	.hash = __VECS(sha512_tv_template)
5426	}
5427	}, {
5428	.alg = "sm3",
5429	.test = alg_test_hash,
5430	.suite = {
5431	.hash = __VECS(sm3_tv_template)
5432	}
5433	}, {
5434	.alg = "streebog256",
5435	.test = alg_test_hash,
5436	.suite = {
5437	.hash = __VECS(streebog256_tv_template)
5438	}
5439	}, {
5440	.alg = "streebog512",
5441	.test = alg_test_hash,
5442	.suite = {
5443	.hash = __VECS(streebog512_tv_template)
5444	}
5445	}, {
5446	.alg = "wp256",
5447	.test = alg_test_hash,
5448	.suite = {
5449	.hash = __VECS(wp256_tv_template)
5450	}
5451	}, {
5452	.alg = "wp384",
5453	.test = alg_test_hash,
5454	.suite = {
5455	.hash = __VECS(wp384_tv_template)
5456	}
5457	}, {
5458	.alg = "wp512",
5459	.test = alg_test_hash,
5460	.suite = {
5461	.hash = __VECS(wp512_tv_template)
5462	}
5463	}, {
5464	.alg = "x962(ecdsa-nist-p192)",
5465	.test = alg_test_sig,
5466	.suite = {
5467	.sig = __VECS(x962_ecdsa_nist_p192_tv_template)
5468	}
5469	}, {
5470	.alg = "x962(ecdsa-nist-p256)",
5471	.test = alg_test_sig,
5472	.fips_allowed = 1,
5473	.suite = {
5474	.sig = __VECS(x962_ecdsa_nist_p256_tv_template)
5475	}
5476	}, {
5477	.alg = "x962(ecdsa-nist-p384)",
5478	.test = alg_test_sig,
5479	.fips_allowed = 1,
5480	.suite = {
5481	.sig = __VECS(x962_ecdsa_nist_p384_tv_template)
5482	}
5483	}, {
5484	.alg = "x962(ecdsa-nist-p521)",
5485	.test = alg_test_sig,
5486	.fips_allowed = 1,
5487	.suite = {
5488	.sig = __VECS(x962_ecdsa_nist_p521_tv_template)
5489	}
5490	}, {
5491	.alg = "xcbc(aes)",
5492	.test = alg_test_hash,
5493	.suite = {
5494	.hash = __VECS(aes_xcbc128_tv_template)
5495	}
5496	}, {
5497	.alg = "xcbc(sm4)",
5498	.test = alg_test_hash,
5499	.suite = {
5500	.hash = __VECS(sm4_xcbc128_tv_template)
5501	}
5502	}, {
5503	.alg = "xchacha12",
5504	.generic_driver = "xchacha12-lib",
5505	.test = alg_test_skcipher,
5506	.suite = {
5507	.cipher = __VECS(xchacha12_tv_template)
5508	},
5509	}, {
5510	.alg = "xchacha20",
5511	.generic_driver = "xchacha20-lib",
5512	.test = alg_test_skcipher,
5513	.suite = {
5514	.cipher = __VECS(xchacha20_tv_template)
5515	},
5516	}, {
5517	.alg = "xctr(aes)",
5518	.test = alg_test_skcipher,
5519	.suite = {
5520	.cipher = __VECS(aes_xctr_tv_template)
5521	}
5522	}, {
5523	.alg = "xts(aes)",
5524	.generic_driver = "xts(ecb(aes-generic))",
5525	.test = alg_test_skcipher,
5526	.fips_allowed = 1,
5527	.suite = {
5528	.cipher = __VECS(aes_xts_tv_template)
5529	}
5530	}, {
5531	.alg = "xts(camellia)",
5532	.generic_driver = "xts(ecb(camellia-generic))",
5533	.test = alg_test_skcipher,
5534	.suite = {
5535	.cipher = __VECS(camellia_xts_tv_template)
5536	}
5537	}, {
5538	.alg = "xts(cast6)",
5539	.generic_driver = "xts(ecb(cast6-generic))",
5540	.test = alg_test_skcipher,
5541	.suite = {
5542	.cipher = __VECS(cast6_xts_tv_template)
5543	}
5544	}, {
5545	/* Same as xts(aes) except the key is stored in
5546	* hardware secure memory which we reference by index
5547	*/
5548	.alg = "xts(paes)",
5549	.test = alg_test_null,
5550	.fips_allowed = 1,
5551	}, {
5552	.alg = "xts(serpent)",
5553	.generic_driver = "xts(ecb(serpent-generic))",
5554	.test = alg_test_skcipher,
5555	.suite = {
5556	.cipher = __VECS(serpent_xts_tv_template)
5557	}
5558	}, {
5559	.alg = "xts(sm4)",
5560	.generic_driver = "xts(ecb(sm4-generic))",
5561	.test = alg_test_skcipher,
5562	.suite = {
5563	.cipher = __VECS(sm4_xts_tv_template)
5564	}
5565	}, {
5566	.alg = "xts(twofish)",
5567	.generic_driver = "xts(ecb(twofish-generic))",
5568	.test = alg_test_skcipher,
5569	.suite = {
5570	.cipher = __VECS(tf_xts_tv_template)
5571	}
5572	}, {
5573	#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5574	.alg = "xts-paes-s390",
5575	.fips_allowed = 1,
5576	.test = alg_test_skcipher,
5577	.suite = {
5578	.cipher = __VECS(aes_xts_tv_template)
5579	}
5580	}, {
5581	#endif
5582	.alg = "xxhash64",
5583	.test = alg_test_hash,
5584	.fips_allowed = 1,
5585	.suite = {
5586	.hash = __VECS(xxhash64_tv_template)
5587	}
5588	}, {
5589	.alg = "zstd",
5590	.test = alg_test_comp,
5591	.fips_allowed = 1,
5592	.suite = {
5593	.comp = {
5594	.comp = __VECS(zstd_comp_tv_template),
5595	.decomp = __VECS(zstd_decomp_tv_template)
5596	}
5597	}
5598	}
5599	};
5600
5601	static void alg_check_test_descs_order(void)
5602	{
5603	int i;
5604
5605	for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
5606	int diff = strcmp(alg_test_descs[i - 1].alg,
5607	alg_test_descs[i].alg);
5608
5609	if (WARN_ON(diff > 0)) {
5610	pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
5611	alg_test_descs[i - 1].alg,
5612	alg_test_descs[i].alg);
5613	}
5614
5615	if (WARN_ON(diff == 0)) {
5616	pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
5617	alg_test_descs[i].alg);
5618	}
5619	}
5620	}
5621
5622	static void alg_check_testvec_configs(void)
5623	{
5624	int i;
5625
5626	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
5627	WARN_ON(!valid_testvec_config(
5628	&default_cipher_testvec_configs[i]));
5629
5630	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
5631	WARN_ON(!valid_testvec_config(
5632	&default_hash_testvec_configs[i]));
5633	}
5634
5635	static void testmgr_onetime_init(void)
5636	{
5637	alg_check_test_descs_order();
5638	alg_check_testvec_configs();
5639
5640	if (!noslowtests)
5641	pr_warn("alg: full crypto tests enabled. This is intended for developer use only.\n");
5642	}
5643
5644	static int alg_find_test(const char *alg)
5645	{
5646	int start = 0;
5647	int end = ARRAY_SIZE(alg_test_descs);
5648
5649	while (start < end) {
5650	int i = (start + end) / 2;
5651	int diff = strcmp(alg_test_descs[i].alg, alg);
5652
5653	if (diff > 0) {
5654	end = i;
5655	continue;
5656	}
5657
5658	if (diff < 0) {
5659	start = i + 1;
5660	continue;
5661	}
5662
5663	return i;
5664	}
5665
5666	return -1;
5667	}
5668
5669	static int alg_fips_disabled(const char *driver, const char *alg)
5670	{
5671	pr_info("alg: %s (%s) is disabled due to FIPS\n", alg, driver);
5672
5673	return -ECANCELED;
5674	}
5675
5676	int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
5677	{
5678	int i;
5679	int j;
5680	int rc;
5681
5682	if (!fips_enabled && notests) {
5683	printk_once(KERN_INFO "alg: self-tests disabled\n");
5684	return 0;
5685	}
5686
5687	DO_ONCE(testmgr_onetime_init);
5688
5689	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
5690	char nalg[CRYPTO_MAX_ALG_NAME];
5691
5692	if (snprintf(buf: nalg, size: sizeof(nalg), fmt: "ecb(%s)", alg) >=
5693	sizeof(nalg))
5694	return -ENAMETOOLONG;
5695
5696	i = alg_find_test(alg: nalg);
5697	if (i < 0)
5698	goto notest;
5699
5700	if (fips_enabled && !alg_test_descs[i].fips_allowed)
5701	goto non_fips_alg;
5702
5703	rc = alg_test_cipher(desc: alg_test_descs + i, driver, type, mask);
5704	goto test_done;
5705	}
5706
5707	i = alg_find_test(alg);
5708	j = alg_find_test(alg: driver);
5709	if (i < 0 && j < 0)
5710	goto notest;
5711
5712	if (fips_enabled) {
5713	if (j >= 0 && !alg_test_descs[j].fips_allowed)
5714	return -EINVAL;
5715
5716	if (i >= 0 && !alg_test_descs[i].fips_allowed)
5717	goto non_fips_alg;
5718	}
5719
5720	rc = 0;
5721	if (i >= 0)
5722	rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
5723	type, mask);
5724	if (j >= 0 && j != i)
5725	rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
5726	type, mask);
5727
5728	test_done:
5729	if (rc) {
5730	if (fips_enabled) {
5731	fips_fail_notify();
5732	panic(fmt: "alg: self-tests for %s (%s) failed in fips mode!\n",
5733	driver, alg);
5734	}
5735	pr_warn("alg: self-tests for %s using %s failed (rc=%d)",
5736	alg, driver, rc);
5737	WARN(rc != -ENOENT,
5738	"alg: self-tests for %s using %s failed (rc=%d)",
5739	alg, driver, rc);
5740	} else {
5741	if (fips_enabled)
5742	pr_info("alg: self-tests for %s (%s) passed\n",
5743	driver, alg);
5744	}
5745
5746	return rc;
5747
5748	notest:
5749	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_LSKCIPHER) {
5750	char nalg[CRYPTO_MAX_ALG_NAME];
5751
5752	if (snprintf(buf: nalg, size: sizeof(nalg), fmt: "ecb(%s)", alg) >=
5753	sizeof(nalg))
5754	goto notest2;
5755
5756	i = alg_find_test(alg: nalg);
5757	if (i < 0)
5758	goto notest2;
5759
5760	if (fips_enabled && !alg_test_descs[i].fips_allowed)
5761	goto non_fips_alg;
5762
5763	rc = alg_test_skcipher(desc: alg_test_descs + i, driver, type, mask);
5764	goto test_done;
5765	}
5766
5767	notest2:
5768	printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);
5769
5770	if (type & CRYPTO_ALG_FIPS_INTERNAL)
5771	return alg_fips_disabled(driver, alg);
5772
5773	return 0;
5774	non_fips_alg:
5775	return alg_fips_disabled(driver, alg);
5776	}
5777
5778	#endif /* CONFIG_CRYPTO_SELFTESTS */
5779
5780	EXPORT_SYMBOL_GPL(alg_test);
5781