Line data Source code
1 : /*
2 : * Copyright (C) 2013 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 2 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : */
14 :
15 : #ifndef __RAID_INTERNAL_H
16 : #define __RAID_INTERNAL_H
17 :
18 : /*
19 : * Supported instruction sets.
20 : *
21 : * It may happen that the assembler is too old to support
22 : * all instructions, even if the architecture supports them.
23 : * These defines allow to exclude from the build the not supported ones.
24 : *
25 : * If in your project you use a predefined assembler, you can define them
26 : * using fixed values, instead of using the HAVE_* defines.
27 : */
28 : #if HAVE_CONFIG_H
29 :
30 : /* Includes the project configuration for HAVE_* defines */
31 : #include "config.h"
32 :
33 : /* If the compiler supports assembly */
34 : #if HAVE_ASSEMBLY
35 : /* Autodetect from the compiler */
36 : #if defined(__i386__)
37 : #define CONFIG_X86 1
38 : #define CONFIG_X86_32 1
39 : #endif
40 : #if defined(__x86_64__)
41 : #define CONFIG_X86 1
42 : #define CONFIG_X86_64 1
43 : #endif
44 : #endif
45 :
46 : /* Enables SSE2, SSSE3, AVX2 only if the assembler supports it */
47 : #if HAVE_SSE2
48 : #define CONFIG_SSE2 1
49 : #endif
50 : #if HAVE_SSSE3
51 : #define CONFIG_SSSE3 1
52 : #endif
53 : #if HAVE_AVX2
54 : #define CONFIG_AVX2 1
55 : #endif
56 :
57 : #else /* if HAVE_CONFIG_H is not defined */
58 :
59 : /* Assume that assembly is always supported */
60 : #if defined(__i386__)
61 : #define CONFIG_X86 1
62 : #define CONFIG_X86_32 1
63 : #endif
64 :
65 : #if defined(__x86_64__)
66 : #define CONFIG_X86 1
67 : #define CONFIG_X86_64 1
68 : #endif
69 :
70 : /* Assumes that the assembler supports everything */
71 : #ifdef CONFIG_X86
72 : #define CONFIG_SSE2 1
73 : #define CONFIG_SSSE3 1
74 : #define CONFIG_AVX2 1
75 : #endif
76 : #endif
77 :
78 : /*
79 : * Includes anything required for compatibility.
80 : */
81 : #include <assert.h>
82 : #include <stdint.h>
83 : #include <stdlib.h>
84 : #include <string.h>
85 :
86 : /*
87 : * Inverse assert.
88 : */
89 : #define BUG_ON(a) assert(!(a))
90 :
91 : /*
92 : * Forced inline.
93 : */
94 : #ifndef __always_inline
95 : #define __always_inline inline __attribute__((always_inline))
96 : #endif
97 :
98 : /*
99 : * Forced alignment.
100 : */
101 : #ifndef __aligned
102 : #define __aligned(a) __attribute__((aligned(a)))
103 : #endif
104 :
105 : /*
106 : * Align a pointer at the specified size.
107 : */
108 : static __always_inline void *__align_ptr(void *ptr, uintptr_t size)
109 : {
110 97079 : uintptr_t offset = (uintptr_t)ptr;
111 :
112 97079 : offset = (offset + size - 1U) & ~(size - 1U);
113 :
114 97079 : return (void *)offset;
115 : }
116 :
117 : /*
118 : * Includes the main interface headers.
119 : */
120 : #include "raid.h"
121 : #include "helper.h"
122 :
123 : /*
124 : * Internal functions.
125 : *
126 : * These are intended to provide access for testing.
127 : */
128 : int raid_selftest(void);
129 : void raid_gen_ref(int nd, int np, size_t size, void **vv);
130 : void raid_invert(uint8_t *M, uint8_t *V, int n);
131 : void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v);
132 : void raid_rec1of1(int *id, int nd, size_t size, void **v);
133 : void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv);
134 : void raid_gen1_int32(int nd, size_t size, void **vv);
135 : void raid_gen1_int64(int nd, size_t size, void **vv);
136 : void raid_gen1_sse2(int nd, size_t size, void **vv);
137 : void raid_gen1_avx2(int nd, size_t size, void **vv);
138 : void raid_gen2_int32(int nd, size_t size, void **vv);
139 : void raid_gen2_int64(int nd, size_t size, void **vv);
140 : void raid_gen2_sse2(int nd, size_t size, void **vv);
141 : void raid_gen2_avx2(int nd, size_t size, void **vv);
142 : void raid_gen2_sse2ext(int nd, size_t size, void **vv);
143 : void raid_genz_int32(int nd, size_t size, void **vv);
144 : void raid_genz_int64(int nd, size_t size, void **vv);
145 : void raid_genz_sse2(int nd, size_t size, void **vv);
146 : void raid_genz_sse2ext(int nd, size_t size, void **vv);
147 : void raid_genz_avx2ext(int nd, size_t size, void **vv);
148 : void raid_gen3_int8(int nd, size_t size, void **vv);
149 : void raid_gen3_ssse3(int nd, size_t size, void **vv);
150 : void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
151 : void raid_gen3_avx2ext(int nd, size_t size, void **vv);
152 : void raid_gen4_int8(int nd, size_t size, void **vv);
153 : void raid_gen4_ssse3(int nd, size_t size, void **vv);
154 : void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
155 : void raid_gen4_avx2ext(int nd, size_t size, void **vv);
156 : void raid_gen5_int8(int nd, size_t size, void **vv);
157 : void raid_gen5_ssse3(int nd, size_t size, void **vv);
158 : void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
159 : void raid_gen5_avx2ext(int nd, size_t size, void **vv);
160 : void raid_gen6_int8(int nd, size_t size, void **vv);
161 : void raid_gen6_ssse3(int nd, size_t size, void **vv);
162 : void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
163 : void raid_gen6_avx2ext(int nd, size_t size, void **vv);
164 : void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
165 : void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
166 : void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
167 : void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
168 : void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
169 : void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
170 : void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
171 : void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
172 : void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
173 :
174 : /*
175 : * Internal naming.
176 : *
177 : * These are intented to provide access for testing.
178 : */
179 : const char *raid_gen1_tag(void);
180 : const char *raid_gen2_tag(void);
181 : const char *raid_genz_tag(void);
182 : const char *raid_gen3_tag(void);
183 : const char *raid_gen4_tag(void);
184 : const char *raid_gen5_tag(void);
185 : const char *raid_gen6_tag(void);
186 : const char *raid_rec1_tag(void);
187 : const char *raid_rec2_tag(void);
188 : const char *raid_recX_tag(void);
189 :
190 : /*
191 : * Internal forwarders.
192 : */
193 : extern void (*raid_gen3_ptr)(int nd, size_t size, void **vv);
194 : extern void (*raid_genz_ptr)(int nd, size_t size, void **vv);
195 : extern void (*raid_gen_ptr[RAID_PARITY_MAX])(
196 : int nd, size_t size, void **vv);
197 : extern void (*raid_rec_ptr[RAID_PARITY_MAX])(
198 : int nr, int *id, int *ip, int nd, size_t size, void **vv);
199 :
200 : /*
201 : * Tables.
202 : */
203 : extern const uint8_t raid_gfmul[256][256] __aligned(256);
204 : extern const uint8_t raid_gfexp[256] __aligned(256);
205 : extern const uint8_t raid_gfinv[256] __aligned(256);
206 : extern const uint8_t raid_gfvandermonde[3][256] __aligned(256);
207 : extern const uint8_t raid_gfcauchy[6][256] __aligned(256);
208 : extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __aligned(256);
209 : extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256);
210 : extern const uint8_t (*raid_gfgen)[256];
211 : #define gfmul raid_gfmul
212 : #define gfexp raid_gfexp
213 : #define gfinv raid_gfinv
214 : #define gfvandermonde raid_gfvandermonde
215 : #define gfcauchy raid_gfcauchy
216 : #define gfgenpshufb raid_gfcauchypshufb
217 : #define gfmulpshufb raid_gfmulpshufb
218 : #define gfgen raid_gfgen
219 :
220 : /*
221 : * Assembler blocks.
222 : */
223 : #ifdef CONFIG_X86
224 : #ifdef CONFIG_SSE2
225 : static __always_inline void raid_sse_begin(void)
226 : {
227 : }
228 :
229 : static __always_inline void raid_sse_end(void)
230 : {
231 : /* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */
232 : /* that use a weak memory model. To ensure that other processors */
233 : /* see correctly the data written, we use a store-store memory */
234 : /* barrier at the end of the asm code */
235 1389322 : asm volatile ("sfence" : : : "memory");
236 :
237 : /* clobbers registers used in the asm code */
238 : /* this is required because in the Windows ABI, */
239 : /* registers xmm6-xmm15 should be kept by the callee. */
240 : /* this clobber list force the compiler to save any */
241 : /* register that needs to be saved */
242 : /* we check for __SSE2_ because we require that the */
243 : /* compiler supports SSE2 registers in the clobber list */
244 : #ifdef __SSE2__
245 1389322 : asm volatile ("" : : : "%xmm0", "%xmm1", "%xmm2", "%xmm3");
246 1389322 : asm volatile ("" : : : "%xmm4", "%xmm5", "%xmm6", "%xmm7");
247 : #ifdef CONFIG_X86_64
248 1389322 : asm volatile ("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11");
249 1389322 : asm volatile ("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15");
250 : #endif
251 : #endif
252 : }
253 : #endif
254 :
255 : #ifdef CONFIG_AVX2
256 : static __always_inline void raid_avx_begin(void)
257 : {
258 : raid_sse_begin();
259 : }
260 :
261 : static __always_inline void raid_avx_end(void)
262 : {
263 : raid_sse_end();
264 :
265 : /* reset the upper part of the ymm registers */
266 : /* to avoid the 70 clocks penality on the next */
267 : /* xmm register use */
268 76334 : asm volatile ("vzeroupper" : : : "memory");
269 : }
270 : #endif
271 : #endif /* CONFIG_X86 */
272 :
273 : #endif
274 :
|
