[go: up one dir, main page]
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Cryptographic API.
4 *
5 * Copyright (c) 2017-present, Facebook, Inc.
6 */
7#include <linux/crypto.h>
8#include <linux/init.h>
9#include <linux/interrupt.h>
10#include <linux/mm.h>
11#include <linux/module.h>
12#include <linux/net.h>
13#include <linux/overflow.h>
14#include <linux/vmalloc.h>
15#include <linux/zstd.h>
16#include <crypto/internal/acompress.h>
17#include <crypto/scatterwalk.h>
18
19
20#define ZSTD_DEF_LEVEL 3
21#define ZSTD_MAX_WINDOWLOG 18
22#define ZSTD_MAX_SIZE BIT(ZSTD_MAX_WINDOWLOG)
23
24struct zstd_ctx {
25 zstd_cctx *cctx;
26 zstd_dctx *dctx;
27 size_t wksp_size;
28 zstd_parameters params;
29 u8 wksp[] __aligned(8) __counted_by(wksp_size);
30};
31
32static DEFINE_MUTEX(zstd_stream_lock);
33
34static void *zstd_alloc_stream(void)
35{
36 zstd_parameters params;
37 struct zstd_ctx *ctx;
38 size_t wksp_size;
39
40 params = zstd_get_params(ZSTD_DEF_LEVEL, ZSTD_MAX_SIZE);
41
42 wksp_size = max(zstd_cstream_workspace_bound(&params.cParams),
43 zstd_dstream_workspace_bound(ZSTD_MAX_SIZE));
44 if (!wksp_size)
45 return ERR_PTR(error: -EINVAL);
46
47 ctx = kvmalloc(struct_size(ctx, wksp, wksp_size), GFP_KERNEL);
48 if (!ctx)
49 return ERR_PTR(error: -ENOMEM);
50
51 ctx->params = params;
52 ctx->wksp_size = wksp_size;
53
54 return ctx;
55}
56
57static void zstd_free_stream(void *ctx)
58{
59 kvfree(addr: ctx);
60}
61
62static struct crypto_acomp_streams zstd_streams = {
63 .alloc_ctx = zstd_alloc_stream,
64 .free_ctx = zstd_free_stream,
65};
66
67static int zstd_init(struct crypto_acomp *acomp_tfm)
68{
69 int ret = 0;
70
71 mutex_lock(&zstd_stream_lock);
72 ret = crypto_acomp_alloc_streams(s: &zstd_streams);
73 mutex_unlock(lock: &zstd_stream_lock);
74
75 return ret;
76}
77
78static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx,
79 const void *src, void *dst, unsigned int *dlen)
80{
81 size_t out_len;
82
83 ctx->cctx = zstd_init_cctx(workspace: ctx->wksp, workspace_size: ctx->wksp_size);
84 if (!ctx->cctx)
85 return -EINVAL;
86
87 out_len = zstd_compress_cctx(cctx: ctx->cctx, dst, dst_capacity: req->dlen, src, src_size: req->slen,
88 parameters: &ctx->params);
89 if (zstd_is_error(code: out_len))
90 return -EINVAL;
91
92 *dlen = out_len;
93
94 return 0;
95}
96
97static int zstd_compress(struct acomp_req *req)
98{
99 struct crypto_acomp_stream *s;
100 unsigned int pos, scur, dcur;
101 unsigned int total_out = 0;
102 bool data_available = true;
103 zstd_out_buffer outbuf;
104 struct acomp_walk walk;
105 zstd_in_buffer inbuf;
106 struct zstd_ctx *ctx;
107 size_t pending_bytes;
108 size_t num_bytes;
109 int ret;
110
111 s = crypto_acomp_lock_stream_bh(s: &zstd_streams);
112 ctx = s->ctx;
113
114 ret = acomp_walk_virt(walk: &walk, req, atomic: true);
115 if (ret)
116 goto out;
117
118 ctx->cctx = zstd_init_cstream(parameters: &ctx->params, pledged_src_size: 0, workspace: ctx->wksp, workspace_size: ctx->wksp_size);
119 if (!ctx->cctx) {
120 ret = -EINVAL;
121 goto out;
122 }
123
124 do {
125 dcur = acomp_walk_next_dst(walk: &walk);
126 if (!dcur) {
127 ret = -ENOSPC;
128 goto out;
129 }
130
131 outbuf.pos = 0;
132 outbuf.dst = (u8 *)walk.dst.virt.addr;
133 outbuf.size = dcur;
134
135 do {
136 scur = acomp_walk_next_src(walk: &walk);
137 if (dcur == req->dlen && scur == req->slen) {
138 ret = zstd_compress_one(req, ctx, src: walk.src.virt.addr,
139 dst: walk.dst.virt.addr, dlen: &total_out);
140 acomp_walk_done_src(walk: &walk, used: scur);
141 acomp_walk_done_dst(walk: &walk, used: dcur);
142 goto out;
143 }
144
145 if (scur) {
146 inbuf.pos = 0;
147 inbuf.src = walk.src.virt.addr;
148 inbuf.size = scur;
149 } else {
150 data_available = false;
151 break;
152 }
153
154 num_bytes = zstd_compress_stream(cstream: ctx->cctx, output: &outbuf, input: &inbuf);
155 if (ZSTD_isError(result: num_bytes)) {
156 ret = -EIO;
157 goto out;
158 }
159
160 pending_bytes = zstd_flush_stream(cstream: ctx->cctx, output: &outbuf);
161 if (ZSTD_isError(result: pending_bytes)) {
162 ret = -EIO;
163 goto out;
164 }
165 acomp_walk_done_src(walk: &walk, used: inbuf.pos);
166 } while (dcur != outbuf.pos);
167
168 total_out += outbuf.pos;
169 acomp_walk_done_dst(walk: &walk, used: dcur);
170 } while (data_available);
171
172 pos = outbuf.pos;
173 num_bytes = zstd_end_stream(cstream: ctx->cctx, output: &outbuf);
174 if (ZSTD_isError(result: num_bytes))
175 ret = -EIO;
176 else
177 total_out += (outbuf.pos - pos);
178
179out:
180 if (ret)
181 req->dlen = 0;
182 else
183 req->dlen = total_out;
184
185 crypto_acomp_unlock_stream_bh(stream: s);
186
187 return ret;
188}
189
190static int zstd_decompress_one(struct acomp_req *req, struct zstd_ctx *ctx,
191 const void *src, void *dst, unsigned int *dlen)
192{
193 size_t out_len;
194
195 ctx->dctx = zstd_init_dctx(workspace: ctx->wksp, workspace_size: ctx->wksp_size);
196 if (!ctx->dctx)
197 return -EINVAL;
198
199 out_len = zstd_decompress_dctx(dctx: ctx->dctx, dst, dst_capacity: req->dlen, src, src_size: req->slen);
200 if (zstd_is_error(code: out_len))
201 return -EINVAL;
202
203 *dlen = out_len;
204
205 return 0;
206}
207
208static int zstd_decompress(struct acomp_req *req)
209{
210 struct crypto_acomp_stream *s;
211 unsigned int total_out = 0;
212 unsigned int scur, dcur;
213 zstd_out_buffer outbuf;
214 struct acomp_walk walk;
215 zstd_in_buffer inbuf;
216 struct zstd_ctx *ctx;
217 size_t pending_bytes;
218 int ret;
219
220 s = crypto_acomp_lock_stream_bh(s: &zstd_streams);
221 ctx = s->ctx;
222
223 ret = acomp_walk_virt(walk: &walk, req, atomic: true);
224 if (ret)
225 goto out;
226
227 ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, workspace: ctx->wksp, workspace_size: ctx->wksp_size);
228 if (!ctx->dctx) {
229 ret = -EINVAL;
230 goto out;
231 }
232
233 do {
234 scur = acomp_walk_next_src(walk: &walk);
235 if (scur) {
236 inbuf.pos = 0;
237 inbuf.size = scur;
238 inbuf.src = walk.src.virt.addr;
239 } else {
240 break;
241 }
242
243 do {
244 dcur = acomp_walk_next_dst(walk: &walk);
245 if (dcur == req->dlen && scur == req->slen) {
246 ret = zstd_decompress_one(req, ctx, src: walk.src.virt.addr,
247 dst: walk.dst.virt.addr, dlen: &total_out);
248 acomp_walk_done_dst(walk: &walk, used: dcur);
249 acomp_walk_done_src(walk: &walk, used: scur);
250 goto out;
251 }
252
253 if (!dcur) {
254 ret = -ENOSPC;
255 goto out;
256 }
257
258 outbuf.pos = 0;
259 outbuf.dst = (u8 *)walk.dst.virt.addr;
260 outbuf.size = dcur;
261
262 pending_bytes = zstd_decompress_stream(dstream: ctx->dctx, output: &outbuf, input: &inbuf);
263 if (ZSTD_isError(result: pending_bytes)) {
264 ret = -EIO;
265 goto out;
266 }
267
268 total_out += outbuf.pos;
269
270 acomp_walk_done_dst(walk: &walk, used: outbuf.pos);
271 } while (inbuf.pos != scur);
272
273 acomp_walk_done_src(walk: &walk, used: scur);
274 } while (ret == 0);
275
276out:
277 if (ret)
278 req->dlen = 0;
279 else
280 req->dlen = total_out;
281
282 crypto_acomp_unlock_stream_bh(stream: s);
283
284 return ret;
285}
286
287static struct acomp_alg zstd_acomp = {
288 .base = {
289 .cra_name = "zstd",
290 .cra_driver_name = "zstd-generic",
291 .cra_flags = CRYPTO_ALG_REQ_VIRT,
292 .cra_module = THIS_MODULE,
293 },
294 .init = zstd_init,
295 .compress = zstd_compress,
296 .decompress = zstd_decompress,
297};
298
299static int __init zstd_mod_init(void)
300{
301 return crypto_register_acomp(alg: &zstd_acomp);
302}
303
304static void __exit zstd_mod_fini(void)
305{
306 crypto_unregister_acomp(alg: &zstd_acomp);
307 crypto_acomp_free_streams(s: &zstd_streams);
308}
309
310module_init(zstd_mod_init);
311module_exit(zstd_mod_fini);
312
313MODULE_LICENSE("GPL");
314MODULE_DESCRIPTION("Zstd Compression Algorithm");
315MODULE_ALIAS_CRYPTO("zstd");
316

source code of linux/crypto/zstd.c