Skip to content

Commit 6833817

Browse files
Ard Biesheuvelherbertx
Ard Biesheuvel
authored and
herbertx
committed
crypto: arm64/aes-blk - move kernel mode neon en/disable into loop
When kernel mode NEON was first introduced on arm64, the preserve and restore of the userland NEON state was completely unoptimized, and involved saving all registers on each call to kernel_neon_begin(), and restoring them on each call to kernel_neon_end(). For this reason, the NEON crypto code that was introduced at the time keeps the NEON enabled throughout the execution of the crypto API methods, which may include calls back into the crypto API that could result in memory allocation or other actions that we should avoid when running with preemption disabled. Since then, we have optimized the kernel mode NEON handling, which now restores lazily (upon return to userland), and so the preserve action is only costly the first time it is called after entering the kernel. So let's put the kernel_neon_begin() and kernel_neon_end() calls around the actual invocations of the NEON crypto code, and run the remainder of the code with kernel mode NEON disabled (and preemption enabled) Note that this requires some reshuffling of the registers in the asm code, because the XTS routines can no longer rely on the registers to retain their contents between invocations. Signed-off-by: Ard Biesheuvel <[email protected]> Signed-off-by: Herbert Xu <[email protected]>
1 parent bd2ad88 commit 6833817

File tree

3 files changed

+97
-102
lines changed

3 files changed

+97
-102
lines changed

‎arch/arm64/crypto/aes-glue.c

+46-49
Original file line numberDiff line numberDiff line change
@@ -64,17 +64,17 @@ MODULE_LICENSE("GPL v2");
6464

6565
/* defined in aes-modes.S */
6666
asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
67-
int rounds, int blocks, int first);
67+
int rounds, int blocks);
6868
asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
69-
int rounds, int blocks, int first);
69+
int rounds, int blocks);
7070

7171
asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
72-
int rounds, int blocks, u8 iv[], int first);
72+
int rounds, int blocks, u8 iv[]);
7373
asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
74-
int rounds, int blocks, u8 iv[], int first);
74+
int rounds, int blocks, u8 iv[]);
7575

7676
asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
77-
int rounds, int blocks, u8 ctr[], int first);
77+
int rounds, int blocks, u8 ctr[]);
7878

7979
asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
8080
int rounds, int blocks, u8 const rk2[], u8 iv[],
@@ -133,102 +133,98 @@ static int ecb_encrypt(struct skcipher_request *req)
133133
{
134134
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
135135
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
136-
int err, first, rounds = 6 + ctx->key_length / 4;
136+
int err, rounds = 6 + ctx->key_length / 4;
137137
struct skcipher_walk walk;
138138
unsigned int blocks;
139139

140-
err = skcipher_walk_virt(&walk, req, true);
140+
err = skcipher_walk_virt(&walk, req, false);
141141

142-
kernel_neon_begin();
143-
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
142+
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
143+
kernel_neon_begin();
144144
aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
145-
(u8 *)ctx->key_enc, rounds, blocks, first);
145+
(u8 *)ctx->key_enc, rounds, blocks);
146+
kernel_neon_end();
146147
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
147148
}
148-
kernel_neon_end();
149149
return err;
150150
}
151151

152152
static int ecb_decrypt(struct skcipher_request *req)
153153
{
154154
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
155155
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
156-
int err, first, rounds = 6 + ctx->key_length / 4;
156+
int err, rounds = 6 + ctx->key_length / 4;
157157
struct skcipher_walk walk;
158158
unsigned int blocks;
159159

160-
err = skcipher_walk_virt(&walk, req, true);
160+
err = skcipher_walk_virt(&walk, req, false);
161161

162-
kernel_neon_begin();
163-
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
162+
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
163+
kernel_neon_begin();
164164
aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
165-
(u8 *)ctx->key_dec, rounds, blocks, first);
165+
(u8 *)ctx->key_dec, rounds, blocks);
166+
kernel_neon_end();
166167
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
167168
}
168-
kernel_neon_end();
169169
return err;
170170
}
171171

172172
static int cbc_encrypt(struct skcipher_request *req)
173173
{
174174
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
175175
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
176-
int err, first, rounds = 6 + ctx->key_length / 4;
176+
int err, rounds = 6 + ctx->key_length / 4;
177177
struct skcipher_walk walk;
178178
unsigned int blocks;
179179

180-
err = skcipher_walk_virt(&walk, req, true);
180+
err = skcipher_walk_virt(&walk, req, false);
181181

182-
kernel_neon_begin();
183-
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
182+
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
183+
kernel_neon_begin();
184184
aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
185-
(u8 *)ctx->key_enc, rounds, blocks, walk.iv,
186-
first);
185+
(u8 *)ctx->key_enc, rounds, blocks, walk.iv);
186+
kernel_neon_end();
187187
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
188188
}
189-
kernel_neon_end();
190189
return err;
191190
}
192191

193192
static int cbc_decrypt(struct skcipher_request *req)
194193
{
195194
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
196195
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
197-
int err, first, rounds = 6 + ctx->key_length / 4;
196+
int err, rounds = 6 + ctx->key_length / 4;
198197
struct skcipher_walk walk;
199198
unsigned int blocks;
200199

201-
err = skcipher_walk_virt(&walk, req, true);
200+
err = skcipher_walk_virt(&walk, req, false);
202201

203-
kernel_neon_begin();
204-
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
202+
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
203+
kernel_neon_begin();
205204
aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
206-
(u8 *)ctx->key_dec, rounds, blocks, walk.iv,
207-
first);
205+
(u8 *)ctx->key_dec, rounds, blocks, walk.iv);
206+
kernel_neon_end();
208207
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
209208
}
210-
kernel_neon_end();
211209
return err;
212210
}
213211

214212
static int ctr_encrypt(struct skcipher_request *req)
215213
{
216214
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
217215
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
218-
int err, first, rounds = 6 + ctx->key_length / 4;
216+
int err, rounds = 6 + ctx->key_length / 4;
219217
struct skcipher_walk walk;
220218
int blocks;
221219

222-
err = skcipher_walk_virt(&walk, req, true);
220+
err = skcipher_walk_virt(&walk, req, false);
223221

224-
first = 1;
225-
kernel_neon_begin();
226222
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
223+
kernel_neon_begin();
227224
aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
228-
(u8 *)ctx->key_enc, rounds, blocks, walk.iv,
229-
first);
225+
(u8 *)ctx->key_enc, rounds, blocks, walk.iv);
230226
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
231-
first = 0;
227+
kernel_neon_end();
232228
}
233229
if (walk.nbytes) {
234230
u8 __aligned(8) tail[AES_BLOCK_SIZE];
@@ -241,12 +237,13 @@ static int ctr_encrypt(struct skcipher_request *req)
241237
*/
242238
blocks = -1;
243239

240+
kernel_neon_begin();
244241
aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
245-
blocks, walk.iv, first);
242+
blocks, walk.iv);
243+
kernel_neon_end();
246244
crypto_xor_cpy(tdst, tsrc, tail, nbytes);
247245
err = skcipher_walk_done(&walk, 0);
248246
}
249-
kernel_neon_end();
250247

251248
return err;
252249
}
@@ -270,16 +267,16 @@ static int xts_encrypt(struct skcipher_request *req)
270267
struct skcipher_walk walk;
271268
unsigned int blocks;
272269

273-
err = skcipher_walk_virt(&walk, req, true);
270+
err = skcipher_walk_virt(&walk, req, false);
274271

275-
kernel_neon_begin();
276272
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
273+
kernel_neon_begin();
277274
aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
278275
(u8 *)ctx->key1.key_enc, rounds, blocks,
279276
(u8 *)ctx->key2.key_enc, walk.iv, first);
277+
kernel_neon_end();
280278
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
281279
}
282-
kernel_neon_end();
283280

284281
return err;
285282
}
@@ -292,16 +289,16 @@ static int xts_decrypt(struct skcipher_request *req)
292289
struct skcipher_walk walk;
293290
unsigned int blocks;
294291

295-
err = skcipher_walk_virt(&walk, req, true);
292+
err = skcipher_walk_virt(&walk, req, false);
296293

297-
kernel_neon_begin();
298294
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
295+
kernel_neon_begin();
299296
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
300297
(u8 *)ctx->key1.key_dec, rounds, blocks,
301298
(u8 *)ctx->key2.key_enc, walk.iv, first);
299+
kernel_neon_end();
302300
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
303301
}
304-
kernel_neon_end();
305302

306303
return err;
307304
}
@@ -425,7 +422,7 @@ static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
425422

426423
/* encrypt the zero vector */
427424
kernel_neon_begin();
428-
aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1, 1);
425+
aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1);
429426
kernel_neon_end();
430427

431428
cmac_gf128_mul_by_x(consts, consts);
@@ -454,8 +451,8 @@ static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
454451
return err;
455452

456453
kernel_neon_begin();
457-
aes_ecb_encrypt(key, ks[0], rk, rounds, 1, 1);
458-
aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2, 0);
454+
aes_ecb_encrypt(key, ks[0], rk, rounds, 1);
455+
aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2);
459456
kernel_neon_end();
460457

461458
return cbcmac_setkey(tfm, key, sizeof(key));

0 commit comments

Comments
 (0)