AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity
17e84a9253
kernel_optimize_test/lib/raid6/recov_ssse3.c
Yuanhan Liu 4f8c55c5ad lib/raid6: build proper files on corresponding arch 
sse and avx2 stuff only exist on x86 arch, and we don't need to build
altivec on x86. And we can do that at lib/raid6/Makefile.

Proposed-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reviewed-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Jim Kukunas <james.t.kukunas@linux.intel.com>
Signed-off-by: NeilBrown <neilb@suse.de>
2012-12-13 19:51:04 +11:00

333 lines
9.1 KiB
C
Raw Blame History

/*
* Copyright (C) 2012 Intel Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; version 2
* of the License.
*/
#include <linux/raid/pq.h>
#include "x86.h"
static int raid6_has_ssse3(void)
{
return boot_cpu_has(X86_FEATURE_XMM) &&
boot_cpu_has(X86_FEATURE_XMM2) &&
boot_cpu_has(X86_FEATURE_SSSE3);
}
static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
u8 *p, *q, *dp, *dq;
const u8 *pbmul; /* P multiplier table for B data */
const u8 *qmul; /* Q multiplier table (for both) */
static const u8 __aligned(16) x0f[16] = {
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
p = (u8 *)ptrs[disks-2];
q = (u8 *)ptrs[disks-1];
/* Compute syndrome with zero for the missing data pages
Use the dead data pages as temporary storage for
delta p and delta q */
dp = (u8 *)ptrs[faila];
ptrs[faila] = (void *)raid6_empty_zero_page;
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
ptrs[failb] = (void *)raid6_empty_zero_page;
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
ptrs[failb] = dq;
ptrs[disks-2] = p;
ptrs[disks-1] = q;
/* Now, pick the proper data tables */
pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
raid6_gfexp[failb]]];
kernel_fpu_begin();
asm volatile("movdqa %0,%%xmm7" : : "m" (x0f[0]));
#ifdef CONFIG_X86_64
asm volatile("movdqa %0,%%xmm6" : : "m" (qmul[0]));
asm volatile("movdqa %0,%%xmm14" : : "m" (pbmul[0]));
asm volatile("movdqa %0,%%xmm15" : : "m" (pbmul[16]));
#endif
/* Now do it... */
while (bytes) {
#ifdef CONFIG_X86_64
/* xmm6, xmm14, xmm15 */
asm volatile("movdqa %0,%%xmm1" : : "m" (q[0]));
asm volatile("movdqa %0,%%xmm9" : : "m" (q[16]));
asm volatile("movdqa %0,%%xmm0" : : "m" (p[0]));
asm volatile("movdqa %0,%%xmm8" : : "m" (p[16]));
asm volatile("pxor %0,%%xmm1" : : "m" (dq[0]));
asm volatile("pxor %0,%%xmm9" : : "m" (dq[16]));
asm volatile("pxor %0,%%xmm0" : : "m" (dp[0]));
asm volatile("pxor %0,%%xmm8" : : "m" (dp[16]));
/* xmm0/8 = px */
asm volatile("movdqa %xmm6,%xmm4");
asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
asm volatile("movdqa %xmm6,%xmm12");
asm volatile("movdqa %xmm5,%xmm13");
asm volatile("movdqa %xmm1,%xmm3");
asm volatile("movdqa %xmm9,%xmm11");
asm volatile("movdqa %xmm0,%xmm2"); /* xmm2/10 = px */
asm volatile("movdqa %xmm8,%xmm10");
asm volatile("psraw $4,%xmm1");
asm volatile("psraw $4,%xmm9");
asm volatile("pand %xmm7,%xmm3");
asm volatile("pand %xmm7,%xmm11");
asm volatile("pand %xmm7,%xmm1");
asm volatile("pand %xmm7,%xmm9");
asm volatile("pshufb %xmm3,%xmm4");
asm volatile("pshufb %xmm11,%xmm12");
asm volatile("pshufb %xmm1,%xmm5");
asm volatile("pshufb %xmm9,%xmm13");
asm volatile("pxor %xmm4,%xmm5");
asm volatile("pxor %xmm12,%xmm13");
/* xmm5/13 = qx */
asm volatile("movdqa %xmm14,%xmm4");
asm volatile("movdqa %xmm15,%xmm1");
asm volatile("movdqa %xmm14,%xmm12");
asm volatile("movdqa %xmm15,%xmm9");
asm volatile("movdqa %xmm2,%xmm3");
asm volatile("movdqa %xmm10,%xmm11");
asm volatile("psraw $4,%xmm2");
asm volatile("psraw $4,%xmm10");
asm volatile("pand %xmm7,%xmm3");
asm volatile("pand %xmm7,%xmm11");
asm volatile("pand %xmm7,%xmm2");
asm volatile("pand %xmm7,%xmm10");
asm volatile("pshufb %xmm3,%xmm4");
asm volatile("pshufb %xmm11,%xmm12");
asm volatile("pshufb %xmm2,%xmm1");
asm volatile("pshufb %xmm10,%xmm9");
asm volatile("pxor %xmm4,%xmm1");
asm volatile("pxor %xmm12,%xmm9");
/* xmm1/9 = pbmul[px] */
asm volatile("pxor %xmm5,%xmm1");
asm volatile("pxor %xmm13,%xmm9");
/* xmm1/9 = db = DQ */
asm volatile("movdqa %%xmm1,%0" : "=m" (dq[0]));
asm volatile("movdqa %%xmm9,%0" : "=m" (dq[16]));
asm volatile("pxor %xmm1,%xmm0");
asm volatile("pxor %xmm9,%xmm8");
asm volatile("movdqa %%xmm0,%0" : "=m" (dp[0]));
asm volatile("movdqa %%xmm8,%0" : "=m" (dp[16]));
bytes -= 32;
p += 32;
q += 32;
dp += 32;
dq += 32;
#else
asm volatile("movdqa %0,%%xmm1" : : "m" (*q));
asm volatile("movdqa %0,%%xmm0" : : "m" (*p));
asm volatile("pxor %0,%%xmm1" : : "m" (*dq));
asm volatile("pxor %0,%%xmm0" : : "m" (*dp));
/* 1 = dq ^ q
* 0 = dp ^ p
*/
asm volatile("movdqa %0,%%xmm4" : : "m" (qmul[0]));
asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
asm volatile("movdqa %xmm1,%xmm3");
asm volatile("psraw $4,%xmm1");
asm volatile("pand %xmm7,%xmm3");
asm volatile("pand %xmm7,%xmm1");
asm volatile("pshufb %xmm3,%xmm4");
asm volatile("pshufb %xmm1,%xmm5");
asm volatile("pxor %xmm4,%xmm5");
asm volatile("movdqa %xmm0,%xmm2"); /* xmm2 = px */
/* xmm5 = qx */
asm volatile("movdqa %0,%%xmm4" : : "m" (pbmul[0]));
asm volatile("movdqa %0,%%xmm1" : : "m" (pbmul[16]));
asm volatile("movdqa %xmm2,%xmm3");
asm volatile("psraw $4,%xmm2");
asm volatile("pand %xmm7,%xmm3");
asm volatile("pand %xmm7,%xmm2");
asm volatile("pshufb %xmm3,%xmm4");
asm volatile("pshufb %xmm2,%xmm1");
asm volatile("pxor %xmm4,%xmm1");
/* xmm1 = pbmul[px] */
asm volatile("pxor %xmm5,%xmm1");
/* xmm1 = db = DQ */
asm volatile("movdqa %%xmm1,%0" : "=m" (*dq));
asm volatile("pxor %xmm1,%xmm0");
asm volatile("movdqa %%xmm0,%0" : "=m" (*dp));
bytes -= 16;
p += 16;
q += 16;
dp += 16;
dq += 16;
#endif
}
kernel_fpu_end();
}
static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
void **ptrs)
{
u8 *p, *q, *dq;
const u8 *qmul; /* Q multiplier table */
static const u8 __aligned(16) x0f[16] = {
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
p = (u8 *)ptrs[disks-2];
q = (u8 *)ptrs[disks-1];
/* Compute syndrome with zero for the missing data page
Use the dead data page as temporary storage for delta q */
dq = (u8 *)ptrs[faila];
ptrs[faila] = (void *)raid6_empty_zero_page;
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
ptrs[disks-1] = q;
/* Now, pick the proper data tables */
qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
kernel_fpu_begin();
asm volatile("movdqa %0, %%xmm7" : : "m" (x0f[0]));
while (bytes) {
#ifdef CONFIG_X86_64
asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
asm volatile("movdqa %0, %%xmm4" : : "m" (dq[16]));
asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
/* xmm3 = q[0] ^ dq[0] */
asm volatile("pxor %0, %%xmm4" : : "m" (q[16]));
asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
/* xmm4 = q[16] ^ dq[16] */
asm volatile("movdqa %xmm3, %xmm6");
asm volatile("movdqa %xmm4, %xmm8");
/* xmm4 = xmm8 = q[16] ^ dq[16] */
asm volatile("psraw $4, %xmm3");
asm volatile("pand %xmm7, %xmm6");
asm volatile("pand %xmm7, %xmm3");
asm volatile("pshufb %xmm6, %xmm0");
asm volatile("pshufb %xmm3, %xmm1");
asm volatile("movdqa %0, %%xmm10" : : "m" (qmul[0]));
asm volatile("pxor %xmm0, %xmm1");
asm volatile("movdqa %0, %%xmm11" : : "m" (qmul[16]));
/* xmm1 = qmul[q[0] ^ dq[0]] */
asm volatile("psraw $4, %xmm4");
asm volatile("pand %xmm7, %xmm8");
asm volatile("pand %xmm7, %xmm4");
asm volatile("pshufb %xmm8, %xmm10");
asm volatile("pshufb %xmm4, %xmm11");
asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
asm volatile("pxor %xmm10, %xmm11");
asm volatile("movdqa %0, %%xmm12" : : "m" (p[16]));
/* xmm11 = qmul[q[16] ^ dq[16]] */
asm volatile("pxor %xmm1, %xmm2");
/* xmm2 = p[0] ^ qmul[q[0] ^ dq[0]] */
asm volatile("pxor %xmm11, %xmm12");
/* xmm12 = p[16] ^ qmul[q[16] ^ dq[16]] */
asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
asm volatile("movdqa %%xmm11, %0" : "=m" (dq[16]));
asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
asm volatile("movdqa %%xmm12, %0" : "=m" (p[16]));
bytes -= 32;
p += 32;
q += 32;
dq += 32;
#else
asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
/* xmm3 = *q ^ *dq */
asm volatile("movdqa %xmm3, %xmm6");
asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
asm volatile("psraw $4, %xmm3");
asm volatile("pand %xmm7, %xmm6");
asm volatile("pand %xmm7, %xmm3");
asm volatile("pshufb %xmm6, %xmm0");
asm volatile("pshufb %xmm3, %xmm1");
asm volatile("pxor %xmm0, %xmm1");
/* xmm1 = qmul[*q ^ *dq */
asm volatile("pxor %xmm1, %xmm2");
/* xmm2 = *p ^ qmul[*q ^ *dq] */
asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
bytes -= 16;
p += 16;
q += 16;
dq += 16;
#endif
}
kernel_fpu_end();
}
const struct raid6_recov_calls raid6_recov_ssse3 = {
.data2 = raid6_2data_recov_ssse3,
.datap = raid6_datap_recov_ssse3,
.valid = raid6_has_ssse3,
#ifdef CONFIG_X86_64
.name = "ssse3x2",
#else
.name = "ssse3x1",
#endif
.priority = 1,
};
View Git Blame Copy Permalink