/*
* include/asm-x86_64/xor.h
*
* Optimized RAID-5 checksumming functions for MMX and SSE.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
*
* You should have received a copy of the GNU General Public License
* (for example /usr/src/linux/COPYING); if not, write to the Free
* Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
/*
* Cache avoiding checksumming functions utilizing KNI instructions
* Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
*/
/*
* Based on
* High-speed RAID5 checksumming functions utilizing SSE instructions.
* Copyright (C) 1998 Ingo Molnar.
*/
/*
* x86-64 changes / gcc fixes from Andi Kleen.
* Copyright 2002 Andi Kleen, SuSE Labs.
*/
typedef struct { unsigned long a,b; } __attribute__((aligned(16))) xmm_store_t;
/* Doesn't use gcc to save the XMM registers, because there is no easy way to
tell it to do a clts before the register saving. */
#define XMMS_SAVE \
asm volatile ( \
"movq %%cr0,%0 ;\n\t" \
"clts ;\n\t" \
"movups %%xmm0,(%1) ;\n\t" \
"movups %%xmm1,0x10(%1) ;\n\t" \
"movups %%xmm2,0x20(%1) ;\n\t" \
"movups %%xmm3,0x30(%1) ;\n\t" \
: "=&r" (cr0) \
: "r" (xmm_save) \
: "memory")
#define XMMS_RESTORE \
asm volatile ( \
"sfence ;\n\t" \
"movups (%1),%%xmm0 ;\n\t" \
"movups 0x10(%1),%%xmm1 ;\n\t" \
"movups 0x20(%1),%%xmm2 ;\n\t" \
"movups 0x30(%1),%%xmm3 ;\n\t" \
"movq %0,%%cr0 ;\n\t" \
: \
: "r" (cr0), "r" (xmm_save) \
: "memory")
#define OFFS(x) "16*("#x")"
#define PF_OFFS(x) "320+16*("#x")"
#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
#define LD(x,y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
#define ST(x,y) " movntdq %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
#define PF5(x) " prefetchnta "PF_OFFS(x)"(%[p6]) ;\n"
#define XO1(x,y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
#define XO2(x,y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
#define XO3(x,y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
#define XO4(x,y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
#define XO5(x,y) " xorps "OFFS(x)"(%[p6]), %%xmm"#y" ;\n"
static void
xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
unsigned int lines = bytes >> 7;
unsigned long cr0;
xmm_store_t xmm_save[4];
XMMS_SAVE;
asm volatile (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
LD(i+1,1) \
PF1(i) \
LD(i+2,2) \
LD(i+3,3) \
PF0(i+4) \
XO1(i,0) \
XO1(i+1,1) \
ST(i,0) \
ST(i+1,1) \
XO1(i+2,2) \
XO1(i+3,3) \
ST(i+2,2) \
ST(i+3,3) \
PF0(0)
" .p2align 4 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
" decl %[cnt]\n"
" leaq 128(%[p1]),%[p1]\n"
" leaq 128(%[p2]),%[p2]\n"
" jnz 1b\n"
: [p1] "+r" (p1), [p2] "+r" (p2), [cnt] "+r" (lines)
:
: "memory");
XMMS_RESTORE;
}
static void
xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
unsigned int lines = bytes >> 7;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
LD(i,0) \
LD(i+1,1) \
XO1(i,0) \
XO1(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
PF2(i) \
PF0(i+4) \
XO1(i+2,2) \
XO1(i+3,3) \
XO2(i,0) \
XO2(i+1,1) \
ST(i,0) \
ST(i+1,1) \
XO2(i+2,2) \
XO2(i+3,3) \
ST(i+2,2) \
ST(i+3,3) \
PF0(0)
" .p2align 4 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
" decl %[cnt]\n"
" leaq 128(%[p1]),%[p1]\n"
" leaq 128(%[p2]),%[p2]\n"
" leaq 128(%[p3]),%[p3]\n"
" jnz 1b"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
:
: "memory");
XMMS_RESTORE;
}
static void
xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
unsigned int lines = bytes >> 7;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
LD(i,0) \
LD(i+1,1) \
XO1(i,0) \
XO1(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
PF2(i) \
XO1(i+2,2) \
XO1(i+3,3) \
PF3(i) \
PF0(i+4) \
XO2(i,0) \
XO2(i+1,1) \
XO2(i+2,2) \
XO2(i+3,3) \
XO3(i,0) \
XO3(i+1,1) \
ST(i,0) \
ST(i+1,1) \
XO3(i+2,2) \
XO3(i+3,3) \
ST(i+2,2) \
ST(i+3,3) \
PF0(0)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
" decl %[cnt]\n"
" leaq 128(%[p1]),%[p1]\n"
" leaq 128(%[p2]),%[p2]\n"
" leaq 128(%[p3]),%[p3]\n"
" leaq 128(%[p4]),%[p4]\n"
" jnz 1b"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
:
: "memory" );
XMMS_RESTORE;
}
static void
xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
unsigned int lines = bytes >> 7;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
LD(i,0) \
LD(i+1,1) \
XO1(i,0) \
XO1(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
PF2(i) \
XO1(i+2,2) \
XO1(i+3,3) \
PF3(i) \
XO2(i,0) \
XO2(i+1,1) \
XO2(i+2,2) \
XO2(i+3,3) \
PF4(i) \
PF0(i+4) \
XO3(i,0) \
XO3(i+1,1) \
XO3(i+2,2) \
XO3(i+3,3) \
XO4(i,0) \
XO4(i+1,1) \
ST(i,0) \
ST(i+1,1) \
XO4(i+2,2) \
XO4(i+3,3) \
ST(i+2,2) \
ST(i+3,3) \
PF0(0)
" .p2align 4 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
" decl %[cnt]\n"
" leaq 128(%[p1]),%[p1]\n"
" leaq 128(%[p2]),%[p2]\n"
" leaq 128(%[p3]),%[p3]\n"
" leaq 128(%[p4]),%[p4]\n"
" leaq 128(%[p5]),%[p5]\n"
" jnz 1b"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4),
[p5] "+r" (p5)
:
: "memory");
XMMS_RESTORE;
}
#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC__MINOR__ >= 3)
#define STORE_NTI(x,mem) __builtin_ia32_movnti(&(mem), (x))
#else
#define STORE_NTI(x,mem) asm("movnti %1,%0" : "=m" (mem) : "r" (x))
#endif
static void
xor_64regs_stream_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
__builtin_prefetch(p1 + 5*64, 0, 0);
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
__builtin_prefetch(p2 + 5*64, 0, 0);
STORE_NTI(d0, p1[0]);
STORE_NTI(d1, p1[1]);
STORE_NTI(d2, p1[2]);
STORE_NTI(d3, p1[3]);
STORE_NTI(d4, p1[4]);
STORE_NTI(d5, p1[5]);
STORE_NTI(d6, p1[6]);
STORE_NTI(d7, p1[7]);
p1 += 8;
p2 += 8;
} while (--lines > 0);
}
static void
xor_64regs_stream_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
__builtin_prefetch(p1 + 5*64, 0, 0);
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
__builtin_prefetch(p2 + 5*64, 0, 0);
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
__builtin_prefetch(p3 + 5*64, 0, 0);
STORE_NTI(d0, p1[0]);
STORE_NTI(d1, p1[1]);
STORE_NTI(d2, p1[2]);
STORE_NTI(d3, p1[3]);
STORE_NTI(d4, p1[4]);
STORE_NTI(d5, p1[5]);
STORE_NTI(d6, p1[6]);
STORE_NTI(d7, p1[7]);
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
}
static void
xor_64regs_stream_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
__builtin_prefetch(p1 + 5*64, 0, 0);
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
__builtin_prefetch(p2 + 5*64, 0, 0);
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
__builtin_prefetch(p3 + 5*64, 0, 0);
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
__builtin_prefetch(p4 + 5*64, 0, 0);
STORE_NTI(d0, p1[0]);
STORE_NTI(d1, p1[1]);
STORE_NTI(d2, p1[2]);
STORE_NTI(d3, p1[3]);
STORE_NTI(d4, p1[4]);
STORE_NTI(d5, p1[5]);
STORE_NTI(d6, p1[6]);
STORE_NTI(d7, p1[7]);
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
}
static void
xor_64regs_stream_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
__builtin_prefetch(p1 + 5*64, 0, 0);
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
__builtin_prefetch(p2 + 5*64, 0, 0);
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
__builtin_prefetch(p3 + 5*64, 0, 0);
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
__builtin_prefetch(p4 + 5*64, 0, 0);
d0 ^= p5[0];
d1 ^= p5[1];
d2 ^= p5[2];
d3 ^= p5[3];
d4 ^= p5[4];
d5 ^= p5[5];
d6 ^= p5[6];
d7 ^= p5[7];
__builtin_prefetch(p5 + 5*64, 0, 0);
STORE_NTI(d0, p1[0]);
STORE_NTI(d1, p1[1]);
STORE_NTI(d2, p1[2]);
STORE_NTI(d3, p1[3]);
STORE_NTI(d4, p1[4]);
STORE_NTI(d5, p1[5]);
STORE_NTI(d6, p1[6]);
STORE_NTI(d7, p1[7]);
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
}
static struct xor_block_template xor_block_sse = {
name: "128byte sse streaming",
do_2: xor_sse_2,
do_3: xor_sse_3,
do_4: xor_sse_4,
do_5: xor_sse_5,
};
static struct xor_block_template xor_block_64regs_stream = {
name: "64byte int streaming",
do_2: xor_64regs_stream_2,
do_3: xor_64regs_stream_3,
do_4: xor_64regs_stream_4,
do_5: xor_64regs_stream_5,
};
/* AK: the speed test is useless: it only tests cache hot */
#undef XOR_TRY_TEMPLATES
#define XOR_TRY_TEMPLATES \
do { \
xor_speed(&xor_block_sse); \
xor_speed(&xor_block_64regs_stream); \
} while (0)
#define XOR_SELECT_TEMPLATE(FASTEST) (FASTEST)
![[BACK]](/icons/back.gif){kind=icon}
Return to xor.h CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [Development] / linux-2.4-xfs / include / asm-x86_64