/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/stringify.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/dwarf.h>
#include <asm/fpu-insn.h>
#define STATE0 %v0
#define STATE1 %v1
#define STATE2 %v2
#define STATE3 %v3
#define COPY0 %v4
#define COPY1 %v5
#define COPY2 %v6
#define COPY3 %v7
#define BEPERM %v19
#define TMP0 %v20
#define TMP1 %v21
#define TMP2 %v22
#define TMP3 %v23
.section .rodata
.balign 32
SYM_DATA_START_LOCAL(chacha20_constants)
.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral
.long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap
SYM_DATA_END(chacha20_constants)
.text
/*
* s390 ChaCha20 implementation meant for vDSO. Produces a given positive
* number of blocks of output with nonce 0, taking an input key and 8-bytes
* counter. Does not spill to the stack.
*
* void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
* const uint8_t *key,
* uint32_t *counter,
* size_t nblocks)
*/
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
CFI_STARTPROC
larl %r1,chacha20_constants
/* COPY0 = "expand 32-byte k" */
VL COPY0,0,,%r1
/* BEPERM = byte selectors for VPERM */
ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148)
/* COPY1,COPY2 = key */
VLM COPY1,COPY2,0,%r3
/* COPY3 = counter || zero nonce */
lg %r3,0(%r4)
VZERO COPY3
VLVGG COPY3,%r3,0
lghi %r1,0
.Lblock:
VLR STATE0,COPY0
VLR STATE1,COPY1
VLR STATE2,COPY2
VLR STATE3,COPY3
lghi %r0,10
.Ldoubleround:
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
VAF STATE0,STATE0,STATE1
VX STATE3,STATE3,STATE0
VERLLF STATE3,STATE3,16
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
VAF STATE2,STATE2,STATE3
VX STATE1,STATE1,STATE2
VERLLF STATE1,STATE1,12
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
VAF STATE0,STATE0,STATE1
VX STATE3,STATE3,STATE0
VERLLF STATE3,STATE3,8
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
VAF STATE2,STATE2,STATE3
VX STATE1,STATE1,STATE2
VERLLF STATE1,STATE1,7
/* STATE1[0,1,2,3] = STATE1[1,2,3,0] */
VSLDB STATE1,STATE1,STATE1,4
/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
VSLDB STATE2,STATE2,STATE2,8
/* STATE3[0,1,2,3] = STATE3[3,0,1,2] */
VSLDB STATE3,STATE3,STATE3,12
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
VAF STATE0,STATE0,STATE1
VX STATE3,STATE3,STATE0
VERLLF STATE3,STATE3,16
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
VAF STATE2,STATE2,STATE3
VX STATE1,STATE1,STATE2
VERLLF STATE1,STATE1,12
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
VAF STATE0,STATE0,STATE1
VX STATE3,STATE3,STATE0
VERLLF STATE3,STATE3,8
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
VAF STATE2,STATE2,STATE3
VX STATE1,STATE1,STATE2
VERLLF STATE1,STATE1,7
/* STATE1[0,1,2,3] = STATE1[3,0,1,2] */
VSLDB STATE1,STATE1,STATE1,12
/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
VSLDB STATE2,STATE2,STATE2,8
/* STATE3[0,1,2,3] = STATE3[1,2,3,0] */
VSLDB STATE3,STATE3,STATE3,4
brctg %r0,.Ldoubleround
/* OUTPUT0 = STATE0 + COPY0 */
VAF STATE0,STATE0,COPY0
/* OUTPUT1 = STATE1 + COPY1 */
VAF STATE1,STATE1,COPY1
/* OUTPUT2 = STATE2 + COPY2 */
VAF STATE2,STATE2,COPY2
/* OUTPUT3 = STATE3 + COPY3 */
VAF STATE3,STATE3,COPY3
ALTERNATIVE \
__stringify( \
/* Convert STATE to little endian and store to OUTPUT */\
VPERM TMP0,STATE0,STATE0,BEPERM; \
VPERM TMP1,STATE1,STATE1,BEPERM; \
VPERM TMP2,STATE2,STATE2,BEPERM; \
VPERM TMP3,STATE3,STATE3,BEPERM; \
VSTM TMP0,TMP3,0,%r2), \
__stringify( \
/* 32 bit wise little endian store to OUTPUT */ \
VSTBRF STATE0,0,,%r2; \
VSTBRF STATE1,16,,%r2; \
VSTBRF STATE2,32,,%r2; \
VSTBRF STATE3,48,,%r2; \
brcl 0,0), \
ALT_FACILITY(148)
/* ++COPY3.COUNTER */
/* alsih %r3,1 */
.insn rilu,0xcc0a00000000,%r3,1
alcr %r3,%r1
VLVGG COPY3,%r3,0
/* OUTPUT += 64, --NBLOCKS */
aghi %r2,64
brctg %r5,.Lblock
/* COUNTER = COPY3.COUNTER */
stg %r3,0(%r4)
/* Zero out potentially sensitive regs */
VZERO STATE0
VZERO STATE1
VZERO STATE2
VZERO STATE3
VZERO COPY1
VZERO COPY2
/* Early exit if TMP0-TMP3 have not been used */
ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148)
VZERO TMP0
VZERO TMP1
VZERO TMP2
VZERO TMP3
br %r14
CFI_ENDPROC
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
Codebase Browser
linux