common_x86.h

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#ifndefCOMMON_X86
#defineCOMMON_X86

#ifndefASSEMBLER

#ifdefC_MSVC
#include<intrin.h>
#endif

#defineMB
#defineWMB
#defineRMB

#ifdefC_SUN
#define__asm__ __asm
#define__volatile__
#endif

staticvoid __inline blas_lock(volatileBLASULONG*address){

intret;

do {
while (*address) {YIELDING;};

#if defined(_MSC_VER) && !defined(__clang__)
// use intrinsic instead of inline assembly
ret=_InterlockedExchange((volatileLONG*)address, 1);
// inline assembly
/*__asm {
 mov eax, address
 mov ebx, 1
 xchg [eax], ebx
 mov ret, ebx
 }*/
#else
 __asm__ __volatile__(
"xchgl %0, %1\n"
 : "=r"(ret), "=m"(*address)
 : "0"(1), "m"(*address)
 : "memory");
#endif

 } while (ret);

}
#defineBLAS_LOCK_DEFINED

static __inline unsigned long longrpcc(void){
#if defined(_MSC_VER) && !defined(__clang__)
return__rdtsc(); // use MSVC intrinsic
#else
unsigned inta, d;

 __asm__ __volatile__ ("rdtsc" : "=a" (a), "=d" (d));

return ((unsigned long long)a+ ((unsigned long long)d << 32));
#endif
};
#defineRPCC_DEFINED

static __inline unsigned longgetstackaddr(void){
#if defined(_MSC_VER) && !defined(__clang__)
return (unsigned long)_ReturnAddress(); // use MSVC intrinsic
#else
unsigned longaddr;

 __asm__ __volatile__ ("mov %%esp, %0"
 : "=r"(addr) : : "memory");

returnaddr;
#endif
};


static __inline long doublesqrt_long(long doubleval) {
#if defined(_MSC_VER) && !defined(__clang__)
returnsqrt(val); // not sure if this will use fsqrt
#else
long doubleresult;

 __asm__ __volatile__ ("fldt %1\n"
"fsqrt\n"
"fstpt %0\n" : "=m" (result) : "m"(val));
returnresult;
#endif
}

#defineSQRT(a) sqrt_long(a)

/* This is due to gcc's bug */
voidcpuid(intop, int*eax, int*ebx, int*ecx, int*edx);

#defineWHEREAMI

static __inline intWhereAmI(void){
inteax, ebx, ecx, edx;
intapicid;

cpuid(1, &eax, &ebx, &ecx, &edx);
apicid=BITMASK(ebx, 24, 0xff);

returnapicid;
}

#ifdefENABLE_SSE_EXCEPTION

#defineIDEBUG_START \
{ \
 unsigned int fp_sse_mode, new_fp_mode; \
 __asm__ __volatile__ ("stmxcsr %0" : "=m" (fp_sse_mode) : ); \
 new_fp_mode = fp_sse_mode & ~0xd00; \
 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (new_fp_mode) );

#defineIDEBUG_END \
 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (fp_sse_mode) ); \
}

#endif

#ifdefXDOUBLE
#defineGET_IMAGE(res) __asm__ __volatile__("fstpt %0" : "=m"(res) : : "memory")
#elif defined(DOUBLE)
#defineGET_IMAGE(res) __asm__ __volatile__("fstpl %0" : "=m"(res) : : "memory")
#else
#defineGET_IMAGE(res) __asm__ __volatile__("fstps %0" : "=m"(res) : : "memory");
#endif

#defineGET_IMAGE_CANCEL __asm__ __volatile__ ("ffree %st")

#ifdefSMP
externunsigned intblas_quick_divide_table[];

static __inline intblas_quickdivide(unsigned intx, unsigned inty){

unsigned intresult;

if (y <= 1) returnx;

#if defined(_MSC_VER) && !defined(__clang__)
result=x/y;
returnresult;
#else
#if (MAX_CPU_NUMBER>64)
if ( y>64) {
result=x/y;
returnresult;
 }
#endif

y=blas_quick_divide_table[y];

 __asm__ __volatile__ ("mull %0" :"=d" (result), "+a"(x): "0" (y));

returnresult;
#endif
}
#endif

#endif

#ifndefPAGESIZE
#definePAGESIZE ( 4 << 10)
#endif
#defineHUGE_PAGESIZE ( 4 << 20)

#defineBUFFER_SIZE (16 << 20)

#defineSEEK_ADDRESS

#if defined(DOUBLE) || defined(XDOUBLE)
#defineMMXLOAD movq
#defineMMXSTORE movq
#else
#defineMMXLOAD movd
#defineMMXSTORE movd
#endif

#if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
//Enable some optimization for barcelona.
#defineBARCELONA_OPTIMIZATION
#endif

#if defined(HAVE_3DNOW)
#defineEMMS femms
#elif defined(HAVE_MMX)
#defineEMMS emms
#endif

#ifndefEMMS
#defineEMMS
#endif

#if defined(CORE2) || defined(PENTIUM4)
#definemovapd movaps
#endif

#defineBRANCH .byte 0x3e
#defineNOBRANCH .byte 0x2e
#definePADDING .byte 0x66;
#defineHALT hlt

#ifndefCOMPLEX
#ifdefXDOUBLE
#defineLOCAL_BUFFER_SIZE QLOCAL_BUFFER_SIZE
#elif defined DOUBLE
#defineLOCAL_BUFFER_SIZE DLOCAL_BUFFER_SIZE
#else
#defineLOCAL_BUFFER_SIZE SLOCAL_BUFFER_SIZE
#endif
#else
#ifdefXDOUBLE
#defineLOCAL_BUFFER_SIZE XLOCAL_BUFFER_SIZE
#elif defined DOUBLE
#defineLOCAL_BUFFER_SIZE ZLOCAL_BUFFER_SIZE
#else
#defineLOCAL_BUFFER_SIZE CLOCAL_BUFFER_SIZE
#endif
#endif

#if defined(OS_WINDOWS)
#ifLOCAL_BUFFER_SIZE>16384
#defineSTACK_TOUCHING \
 movl $0, 4096 * 4(%esp);\
 movl $0, 4096 * 3(%esp);\
 movl $0, 4096 * 2(%esp);\
 movl $0, 4096 * 1(%esp);
#elifLOCAL_BUFFER_SIZE>12288
#defineSTACK_TOUCHING \
 movl $0, 4096 * 3(%esp);\
 movl $0, 4096 * 2(%esp);\
 movl $0, 4096 * 1(%esp);
#elifLOCAL_BUFFER_SIZE>8192
#defineSTACK_TOUCHING \
 movl $0, 4096 * 2(%esp);\
 movl $0, 4096 * 1(%esp);
#elifLOCAL_BUFFER_SIZE>4096
#defineSTACK_TOUCHING \
 movl $0, 4096 * 1(%esp);
#else
#defineSTACK_TOUCHING
#endif
#else
#defineSTACK_TOUCHING
#endif

#ifndefF_INTERFACE
#defineREALNAME ASMNAME
#else
#defineREALNAME ASMFNAME
#endif

#if defined(F_INTERFACE_PATHSCALE) || defined(F_INTERFACE_OPEN64)
#defineRETURN_BY_STRUCT
#elif defined(F_INTERFACE_GFORT) || defined(F_INTERFACE_G95)
#defineRETURN_BY_COMPLEX
#else
#defineRETURN_BY_STACK
#endif

#ifdefOS_DARWIN
#definePROLOGUE .text;.align 5; .globl REALNAME; REALNAME:
#defineEPILOGUE .subsections_via_symbols
#definePROFCODE
#endif

#if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)
#defineSAVEREGISTERS \
 subl $32, %esp;\
 movups %xmm6, 0(%esp);\
 movups %xmm7, 16(%esp)

#defineRESTOREREGISTERS \
 movups 0(%esp), %xmm6;\
 movups 16(%esp), %xmm7;\
 addl $32, %esp
#else
#defineSAVEREGISTERS
#defineRESTOREREGISTERS
#endif

#if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)
#definePROLOGUE \
 .text; \
 .align 16; \
 .globl REALNAME ;\
 .def REALNAME;.scl 2;.type 32;.endef; \
REALNAME:

#definePROFCODE

#ifdef__clang__
#defineEPILOGUE .end
#else
#defineEPILOGUE .end REALNAME
#endif
#endif

#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(__ELF__)
#definePROLOGUE \
 .text; \
 .align 16; \
 .globl REALNAME ;\
 .type REALNAME, @function; \
REALNAME: \
 _CET_ENDBR

#ifdefPROFILE
#definePROFCODE call mcount
#else
#definePROFCODE
#endif

#defineEPILOGUE \
 .size REALNAME, .-REALNAME; \
 .section .note.GNU-stack,"",@progbits

#endif

#ifdefXDOUBLE
#defineFLD fldt
#defineFST fstpt
#defineFSTU fstt
#defineFMUL fmult
#defineFADD faddt
#defineFSUB fsubt
#defineFSUBR fsubrt
#elif defined(DOUBLE)
#defineFLD fldl
#defineFST fstpl
#defineFSTU fstl
#defineFMUL fmull
#defineFADD faddl
#defineFSUB fsubl
#defineFSUBR fsubrl
#else
#defineFLD flds
#defineFST fstps
#defineFSTU fsts
#defineFMUL fmuls
#defineFADD fadds
#defineFSUB fsubs
#defineFSUBR fsubrs
#endif
#endif

#ifdefC_SUN
#defineffreep fstp
#endif

#ifdef__APPLE__
#defineALIGN_2 .align 2
#defineALIGN_3 .align 3
#defineALIGN_4 .align 4
#defineALIGN_5 .align 5
#defineffreep fstp
#endif

#ifndefALIGN_2
#defineALIGN_2 .align 4
#endif

#ifndefALIGN_3
#defineALIGN_3 .align 8
#endif

#ifndefALIGN_4
#defineALIGN_4 .align 16
#endif

#ifndefALIGN_5
#defineALIGN_5 .align 32
#endif

#ifndefALIGN_6
#defineALIGN_6 .align 64
#endif
// ffreep %st(0).
// Because Clang didn't support ffreep, we directly use the opcode.
// Please check out http://www.sandpile.org/x86/opc_fpu.htm
#ifndefffreep
#defineffreep .byte 0xdf, 0xc0 #
#endif