common_x86_64.h

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#ifndefCOMMON_X86
#defineCOMMON_X86

#ifndefASSEMBLER

#ifdefC_MSVC
#include<intrin.h>
#endif

#ifdefC_SUN
#define__asm__ __asm
#define__volatile__
#endif

/*
#ifdef HAVE_SSE2
#define MB __asm__ __volatile__ ("mfence");
#define WMB __asm__ __volatile__ ("sfence");
#else
#define MB
#define WMB
#endif
*/

#ifdef__GNUC__
#defineMB do { __asm__ __volatile__("": : :"memory"); } while (0)
#defineWMB do { __asm__ __volatile__("": : :"memory"); } while (0)
#defineRMB
#else
#defineMB do {} while (0)
#defineWMB do {} while (0)
#defineRMB
#endif

staticvoid __inline blas_lock(volatileBLASULONG*address){


#ifndefC_MSVC
intret;
#else
BLASULONGret;
#endif

do {
while (*address) {YIELDING;}

#ifndefC_MSVC
 __asm__ __volatile__(
"xchgl %0, %1\n"
 : "=r"(ret), "=m"(*address)
 : "0"(1), "m"(*address)
 : "memory");
#else
ret=InterlockedExchange64((volatileLONG64*)(address), 1);
#endif
 } while (ret);

}
#defineBLAS_LOCK_DEFINED

static __inline BLASULONGrpcc(void){
#ifdefC_MSVC
return__rdtsc();
#else
BLASULONGa, d;

 __asm__ __volatile__ ("rdtsc" : "=a" (a), "=d" (d));

return ((BLASULONG)a+ ((BLASULONG)d << 32));
#endif
}
#defineRPCC_DEFINED

#defineRPCC64BIT

#ifndefC_MSVC
static __inline BLASULONGgetstackaddr(void){
BLASULONGaddr;

 __asm__ __volatile__ ("movq %%rsp, %0"
 : "=r"(addr) : : "memory");

returnaddr;
}
#endif

static __inline voidcpuid(intop, int*eax, int*ebx, int*ecx, int*edx){

#ifdefC_MSVC
intcpuinfo[4];
__cpuid(cpuinfo, op);
*eax=cpuinfo[0];
*ebx=cpuinfo[1];
*ecx=cpuinfo[2];
*edx=cpuinfo[3];
#else
 __asm__ __volatile__("mov $0, %%ecx;"
"cpuid"
 : "=a" (*eax),
"=b" (*ebx),
"=c" (*ecx),
"=d" (*edx)
 : "0" (op));
#endif
}

static __inline voidcpuid_count(intop, intcount, int*eax, int*ebx, int*ecx, int*edx)
{
#ifdefC_MSVC
intcpuInfo[4] = {-1};
__cpuidex(cpuInfo, op, count);
*eax=cpuInfo[0];
*ebx=cpuInfo[1];
*ecx=cpuInfo[2];
*edx=cpuInfo[3];
#else
#if defined(__i386__) && defined(__PIC__)
 __asm__ __volatile__
 ("mov %%ebx, %%edi;"
"cpuid;"
"xchgl %%ebx, %%edi;"
 : "=a" (*eax), "=D" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (op), "2" (count) : "cc");
#else
 __asm__ __volatile__
 ("cpuid": "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (op), "2" (count) : "cc");
#endif
#endif
}

/*
#define WHEREAMI
*/

static __inline intWhereAmI(void){
inteax, ebx, ecx, edx;
intapicid;

cpuid(1, &eax, &ebx, &ecx, &edx);
apicid=BITMASK(ebx, 24, 0xff);

returnapicid;
}


#ifdefCORE_BARCELONA
#defineIFLUSH gotoblas_iflush()
#defineIFLUSH_HALF gotoblas_iflush_half()
#endif

#ifdefENABLE_SSE_EXCEPTION

#defineIDEBUG_START \
{ \
 unsigned int fp_sse_mode, new_fp_mode; \
 __asm__ __volatile__ ("stmxcsr %0" : "=m" (fp_sse_mode) : ); \
 new_fp_mode = fp_sse_mode & ~0xd00; \
 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (new_fp_mode) );

#defineIDEBUG_END \
 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (fp_sse_mode) ); \
}

#endif

#ifdefXDOUBLE
#defineGET_IMAGE(res) __asm__ __volatile__("fstpt %0" : "=m"(res) : : "memory")
#elif defined(DOUBLE)
#defineGET_IMAGE(res) __asm__ __volatile__("movsd %%xmm1, %0" : "=m"(res) : : "memory")
#else
#defineGET_IMAGE(res) __asm__ __volatile__("movss %%xmm1, %0" : "=m"(res) : : "memory")
#endif

#defineGET_IMAGE_CANCEL

#ifdefSMP
#if defined(USE64BITINT)
static __inline blasintblas_quickdivide(blasintx, blasinty){
returnx / y;
}
#elif defined (C_MSVC)
static __inline BLASLONGblas_quickdivide(BLASLONGx, BLASLONGy){
returnx / y;
}
#else
externunsigned intblas_quick_divide_table[];

static __inline unsigned intblas_quickdivide(unsigned intx, unsigned inty){

volatileunsigned intresult;

if (y <= 1) returnx;

#if (MAX_CPU_NUMBER>64) 
if (y>64) { 
result=x / y;
returnresult;
 }
#endif

y=blas_quick_divide_table[y];

 __asm__ __volatile__ ("mull %0" :"=d" (result), "+a"(x) : "0" (y));
returnresult;
}
#endif
#endif

#endif

#ifndefPAGESIZE
#definePAGESIZE ( 4 << 10)
#endif
#defineHUGE_PAGESIZE ( 2 << 20)

#ifndefBUFFERSIZE
#defineBUFFER_SIZE (32 << 22)
#else
#defineBUFFER_SIZE (32 << BUFFERSIZE)
#endif

#defineSEEK_ADDRESS

#ifdefF_INTERFACE_G77
#defineRETURN_BY_STACK
#defineNEED_F2CCONV
#endif

#ifdefF_INTERFACE_G95
#defineRETURN_BY_PACKED
#endif

#ifdefF_INTERFACE_GFORT
#ifdefOS_WINDOWS
#ifndefDOUBLE
#defineRETURN_BY_REGS
#else
#defineRETURN_BY_STACK
#endif
#else
#defineRETURN_BY_PACKED
#endif
#endif

#ifdefF_INTERFACE_INTEL
#defineRETURN_BY_STACK
#endif

#ifdefF_INTERFACE_FUJITSU
#defineRETURN_BY_STACK
#endif

#ifdefF_INTERFACE_FLANG
#defineRETURN_BY_STACK
#endif

#ifdefF_INTERFACE_PGI
#defineRETURN_BY_STACK
#endif

#ifdefF_INTERFACE_PATHSCALE
#defineRETURN_BY_PACKED
#endif

#ifdefF_INTERFACE_SUN
#defineRETURN_BY_PACKED
#endif

#ifdefASSEMBLER

#if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
//Enable some optimization for barcelona.
#defineBARCELONA_OPTIMIZATION
#endif

#if defined(HAVE_3DNOW)
#defineEMMS femms
#elif defined(HAVE_MMX)
#defineEMMS emms
#endif

#ifndefEMMS
#defineEMMS
#endif

#defineBRANCH .byte 0x3e
#defineNOBRANCH .byte 0x2e
#definePADDING .byte 0x66

#ifdefOS_WINDOWS
#defineARG1 %rcx
#defineARG2 %rdx
#defineARG3 %r8
#defineARG4 %r9
#else
#defineARG1 %rdi
#defineARG2 %rsi
#defineARG3 %rdx
#defineARG4 %rcx
#defineARG5 %r8
#defineARG6 %r9
#endif

#ifndefCOMPLEX
#ifdefXDOUBLE
#defineLOCAL_BUFFER_SIZE QLOCAL_BUFFER_SIZE
#elif defined DOUBLE
#defineLOCAL_BUFFER_SIZE DLOCAL_BUFFER_SIZE
#else
#defineLOCAL_BUFFER_SIZE SLOCAL_BUFFER_SIZE
#endif
#else
#ifdefXDOUBLE
#defineLOCAL_BUFFER_SIZE XLOCAL_BUFFER_SIZE
#elif defined DOUBLE
#defineLOCAL_BUFFER_SIZE ZLOCAL_BUFFER_SIZE
#else
#defineLOCAL_BUFFER_SIZE CLOCAL_BUFFER_SIZE
#endif
#endif

#if defined(OS_WINDOWS)
#ifLOCAL_BUFFER_SIZE>16384
#defineSTACK_TOUCHING \
 movl $0, 4096 * 4(%rsp);\
 movl $0, 4096 * 3(%rsp);\
 movl $0, 4096 * 2(%rsp);\
 movl $0, 4096 * 1(%rsp);
#elifLOCAL_BUFFER_SIZE>12288
#defineSTACK_TOUCHING \
 movl $0, 4096 * 3(%rsp);\
 movl $0, 4096 * 2(%rsp);\
 movl $0, 4096 * 1(%rsp);
#elifLOCAL_BUFFER_SIZE>8192
#defineSTACK_TOUCHING \
 movl $0, 4096 * 2(%rsp);\
 movl $0, 4096 * 1(%rsp);
#elifLOCAL_BUFFER_SIZE>4096
#defineSTACK_TOUCHING \
 movl $0, 4096 * 1(%rsp);
#else
#defineSTACK_TOUCHING
#endif
#else
#defineSTACK_TOUCHING
#endif

#if defined(CORE2)
#definemovapd movaps
#defineandpd andps
#definemovlpd movlps
#definemovhpd movhps
#endif

#ifndefF_INTERFACE
#defineREALNAME ASMNAME
#else
#defineREALNAME ASMFNAME
#endif

#ifdefOS_DARWIN
#definePROLOGUE .text;.align 5; .globl REALNAME; REALNAME:
#defineEPILOGUE .subsections_via_symbols
#definePROFCODE
#endif

#ifdefOS_WINDOWS
#defineSAVEREGISTERS \
 subq $256, %rsp;\
 movups %xmm6, 0(%rsp);\
 movups %xmm7, 16(%rsp);\
 movups %xmm8, 32(%rsp);\
 movups %xmm9, 48(%rsp);\
 movups %xmm10, 64(%rsp);\
 movups %xmm11, 80(%rsp);\
 movups %xmm12, 96(%rsp);\
 movups %xmm13, 112(%rsp);\
 movups %xmm14, 128(%rsp);\
 movups %xmm15, 144(%rsp)

#defineRESTOREREGISTERS \
 movups 0(%rsp), %xmm6;\
 movups 16(%rsp), %xmm7;\
 movups 32(%rsp), %xmm8;\
 movups 48(%rsp), %xmm9;\
 movups 64(%rsp), %xmm10;\
 movups 80(%rsp), %xmm11;\
 movups 96(%rsp), %xmm12;\
 movups 112(%rsp), %xmm13;\
 movups 128(%rsp), %xmm14;\
 movups 144(%rsp), %xmm15;\
 addq $256, %rsp
#else
#defineSAVEREGISTERS
#defineRESTOREREGISTERS
#endif

#if defined(OS_WINDOWS) && !defined(C_PGI)
#definePROLOGUE \
 .text; \
 .align 16; \
 .globl REALNAME ;\
 .def REALNAME;.scl 2;.type 32;.endef; \
REALNAME:

#definePROFCODE

#defineEPILOGUE .end
#endif

#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(__ELF__) || defined(C_PGI)
#definePROLOGUE \
 .text; \
 .align 512; \
 .globl REALNAME ;\
 .type REALNAME, @function; \
REALNAME: \
 _CET_ENDBR

#ifdefPROFILE
#definePROFCODE call *mcount@GOTPCREL(%rip)
#else
#definePROFCODE
#endif

#defineEPILOGUE \
 .size REALNAME, .-REALNAME; \
 .section .note.GNU-stack,"",@progbits


#endif

#endif

#ifdefXDOUBLE
#defineFLD fldt
#defineFST fstpt
#defineMOVQ movq
#elif defined(DOUBLE)
#defineFLD fldl
#defineFST fstpl
#defineFSTU fstl
#defineFMUL fmull
#defineFADD faddl
#defineMOVSD movsd
#defineMULSD mulsd
#defineMULPD mulpd
#defineCMPEQPD cmpeqpd
#defineCOMISD comisd
#definePSRLQ psrlq
#defineANDPD andpd
#defineADDPD addpd
#defineADDSD addsd
#defineSUBPD subpd
#defineSUBSD subsd
#defineMOVQ movq
#defineMOVUPD movupd
#defineXORPD xorpd
#else
#defineFLD flds
#defineFST fstps
#defineFSTU fsts
#defineFMUL fmuls
#defineFADD fadds
#defineMOVSD movss
#defineMULSD mulss
#defineMULPD mulps
#defineCMPEQPD cmpeqps
#defineCOMISD comiss
#definePSRLQ psrld
#defineANDPD andps
#defineADDPD addps
#defineADDSD addss
#defineSUBPD subps
#defineSUBSD subss
#defineMOVQ movd
#defineMOVUPD movups
#defineXORPD xorps
#endif

#defineHALT hlt

#ifdefOS_DARWIN
#defineALIGN_2 .align 2
#defineALIGN_3 .align 3
#defineALIGN_4 .align 4
#defineALIGN_5 .align 5
#defineffreep fstp
#endif

#ifndefALIGN_2
#defineALIGN_2 .align 4
#endif

#ifndefALIGN_3
#defineALIGN_3 .align 8
#endif

#ifndefALIGN_4
#defineALIGN_4 .align 16
#endif

#ifndefALIGN_5
#defineALIGN_5 .align 32
#endif

#ifndefALIGN_6
#defineALIGN_6 .align 64
#endif

// ffreep %st(0).
// Because Clang didn't support ffreep, we directly use the opcode.
// Please check out http://www.sandpile.org/x86/opc_fpu.htm
#ifndefffreep
#defineffreep .byte 0xdf, 0xc0 #
#endif
#endif