1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// ChaCha8 is ChaCha with 8 rounds.
// See https://cr.yp.to/chacha/chacha-20080128.pdf.
// See chacha8_generic.go for additional details.
// ROL rotates the uint32s in register R left by N bits, using temporary T.
#define ROL(N, R, T) \
MOVO R, T; PSLLL $(N), T; PSRLL $(32-(N)), R; PXOR T, R
// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
#ifdef GOAMD64_v2
#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
#else
#define ROL16(R, T) ROL(16, R, T)
#endif
// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
#ifdef GOAMD64_v2
#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
#else
#define ROL8(R, T) ROL(8, R, T)
#endif
// QR is the ChaCha quarter-round on A, B, C, and D. T is an available temporary.
#define QR(A, B, C, D, T) \
PADDD B, A; PXOR A, D; ROL16(D, T); \
PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $12, T; PSRLL $20, B; PXOR T, B; \
PADDD B, A; PXOR A, D; ROL8(D, T); \
PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B
// REPLREG replicates the register R into 4 uint32s in XR.
#define REPLREG(R, XR) \
MOVQ R, XR; \
PSHUFD $0, XR, XR
// REPL replicates the uint32 constant val into 4 uint32s in XR. It smashes DX.
#define REPL(val, XR) \
MOVL $val, DX; \
REPLREG(DX, XR)
// SEED copies the off'th uint32 of the seed into the register XR,
// replicating it into all four stripes of the register.
#define SEED(off, reg, XR) \
MOVL (4*off)(AX), reg; \
REPLREG(reg, XR) \
// block runs 4 ChaCha8 block transformations in the four stripes of the X registers.
// func block(seed *[8]uint32, blocks *[16][4]uint32, counter uint32)
TEXT ·block<ABIInternal>(SB), NOSPLIT, $16
// seed in AX
// blocks in BX
// counter in CX
// Load initial constants into top row.
REPL(0x61707865, X0)
REPL(0x3320646e, X1)
REPL(0x79622d32, X2)
REPL(0x6b206574, X3)
// Load counter into bottom left cell.
// Each stripe gets a different counter: 0, 1, 2, 3.
// (PINSRD is not available in GOAMD64_v1,
// so just do it in memory on all systems.
// This is not on the critical path.)
MOVL CX, 0(SP)
INCL CX
MOVL CX, 4(SP)
INCL CX
MOVL CX, 8(SP)
INCL CX
MOVL CX, 12(SP)
MOVOU 0(SP), X12
// Load seed words into next two rows and into DI, SI, R8..R13
SEED(0, DI, X4)
SEED(1, SI, X5)
SEED(2, R8, X6)
SEED(3, R9, X7)
SEED(4, R10, X8)
SEED(5, R11, X9)
SEED(6, R12, X10)
SEED(7, R13, X11)
// Zeros for remaining two matrix entries.
// We have just enough XMM registers to hold the state,
// without one for the temporary, so we flush and restore
// some values to and from memory to provide a temporary.
// The initial temporary is X15, so zero its memory instead
// of X15 itself.
MOVL $0, DX
MOVQ DX, X13
MOVQ DX, X14
MOVOU X14, (15*16)(BX)
// 4 iterations. Each iteration is 8 quarter-rounds.
MOVL $4, DX
loop:
QR(X0, X4, X8, X12, X15)
MOVOU X4, (4*16)(BX) // save X4
QR(X1, X5, X9, X13, X15)
MOVOU (15*16)(BX), X15 // reload X15; temp now X4
QR(X2, X6, X10, X14, X4)
QR(X3, X7, X11, X15, X4)
QR(X0, X5, X10, X15, X4)
MOVOU X15, (15*16)(BX) // save X15
QR(X1, X6, X11, X12, X4)
MOVOU (4*16)(BX), X4 // reload X4; temp now X15
QR(X2, X7, X8, X13, X15)
QR(X3, X4, X9, X14, X15)
DECL DX
JNZ loop
// Store interlaced blocks back to output buffer,
// adding original seed along the way.
// First the top and bottom rows.
MOVOU X0, (0*16)(BX)
MOVOU X1, (1*16)(BX)
MOVOU X2, (2*16)(BX)
MOVOU X3, (3*16)(BX)
MOVOU X12, (12*16)(BX)
MOVOU X13, (13*16)(BX)
MOVOU X14, (14*16)(BX)
// X15 has already been stored.
// Now we have X0-X3, X12-X15 available for temporaries.
// Add seed rows back to output. We left seed in DI, SI, R8..R13 above.
REPLREG(DI, X0)
REPLREG(SI, X1)
REPLREG(R8, X2)
REPLREG(R9, X3)
REPLREG(R10, X12)
REPLREG(R11, X13)
REPLREG(R12, X14)
REPLREG(R13, X15)
PADDD X0, X4
PADDD X1, X5
PADDD X2, X6
PADDD X3, X7
PADDD X12, X8
PADDD X13, X9
PADDD X14, X10
PADDD X15, X11
MOVOU X4, (4*16)(BX)
MOVOU X5, (5*16)(BX)
MOVOU X6, (6*16)(BX)
MOVOU X7, (7*16)(BX)
MOVOU X8, (8*16)(BX)
MOVOU X9, (9*16)(BX)
MOVOU X10, (10*16)(BX)
MOVOU X11, (11*16)(BX)
MOVL $0, AX
MOVQ AX, X15 // must be 0 on return
RET
// rotate left 16 indexes for PSHUFB
GLOBL ·rol16<>(SB), NOPTR|RODATA, $16
DATA ·rol16<>+0(SB)/8, $0x0504070601000302
DATA ·rol16<>+8(SB)/8, $0x0D0C0F0E09080B0A
// rotate left 8 indexes for PSHUFB
GLOBL ·rol8<>(SB), NOPTR|RODATA, $16
DATA ·rol8<>+0(SB)/8, $0x0605040702010003
DATA ·rol8<>+8(SB)/8, $0x0E0D0C0F0A09080B
|
