Use SSE when available for floating-point operations (bug 5841, r=her).
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/public/jit/x86/assembler-x86.cpp Thu Aug 08 20:26:36 2013 -0700
1.3 @@ -0,0 +1,33 @@
1.4 +/**
1.5 + * vim: set ts=8 sts=2 sw=2 tw=99 et:
1.6 + * =============================================================================
1.7 + * SourcePawn JIT SDK
1.8 + * Copyright (C) 2004-2008 AlliedModders LLC. All rights reserved.
1.9 + * =============================================================================
1.10 + *
1.11 + * This program is free software; you can redistribute it and/or modify it under
1.12 + * the terms of the GNU General Public License, version 3.0, as published by the
1.13 + * Free Software Foundation.
1.14 + *
1.15 + * This program is distributed in the hope that it will be useful, but WITHOUT
1.16 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.17 + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
1.18 + * details.
1.19 + *
1.20 + * You should have received a copy of the GNU General Public License along with
1.21 + * this program. If not, see <http://www.gnu.org/licenses/>.
1.22 + *
1.23 + * As a special exception, AlliedModders LLC gives you permission to link the
1.24 + * code of this program (as well as its derivative works) to "Half-Life 2," the
1.25 + * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
1.26 + * by the Valve Corporation. You must obey the GNU General Public License in
1.27 + * all respects for all other code used. Additionally, AlliedModders LLC grants
1.28 + * this exception to all derivative works. AlliedModders LLC defines further
1.29 + * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
1.30 + * or <http://www.sourcemod.net/license.php>.
1.31 + *
1.32 + * Version: $Id$
1.33 + */
1.34 +#include <assembler-x86.h>
1.35 +
1.36 +CPUFeatures AssemblerX86::X86Features;
2.1 --- a/public/jit/x86/assembler-x86.h Thu Aug 08 20:21:37 2013 -0700
2.2 +++ b/public/jit/x86/assembler-x86.h Thu Aug 08 20:26:36 2013 -0700
2.3 @@ -94,6 +94,25 @@
2.4 }
2.5 };
2.6
2.7 +struct CPUFeatures
2.8 +{
2.9 + CPUFeatures()
2.10 + {
2.11 + memset(this, 0, sizeof(*this));
2.12 + }
2.13 +
2.14 + bool fpu;
2.15 + bool mmx;
2.16 + bool sse;
2.17 + bool sse2;
2.18 + bool sse3;
2.19 + bool ssse3;
2.20 + bool sse4_1;
2.21 + bool sse4_2;
2.22 + bool avx;
2.23 + bool avx2;
2.24 +};
2.25 +
2.26 const Register eax = { 0 };
2.27 const Register ecx = { 1 };
2.28 const Register edx = { 2 };
2.29 @@ -299,7 +318,19 @@
2.30
2.31 class AssemblerX86 : public Assembler
2.32 {
2.33 + private:
2.34 + // List of processor features; to be used, this must be filled in at
2.35 + // startup.
2.36 + static CPUFeatures X86Features;
2.37 +
2.38 public:
2.39 + static void SetFeatures(const CPUFeatures &features) {
2.40 + X86Features = features;
2.41 + }
2.42 + static const CPUFeatures &Features() {
2.43 + return X86Features;
2.44 + }
2.45 +
2.46 void movl(Register dest, Register src) {
2.47 emit1(0x89, src.code, dest.code);
2.48 }
2.49 @@ -712,6 +743,74 @@
2.50 outOfMemory_ = true;
2.51 }
2.52
2.53 + void cpuid() {
2.54 + emit2(0x0f, 0xa2);
2.55 + }
2.56 +
2.57 +
2.58 + // SSE operations can only be used if the feature detection function has
2.59 + // been run *and* detected the appropriate level of functionality.
2.60 + void movss(FloatRegister dest, const Operand &src) {
2.61 + assert(Features().sse);
2.62 + emit3(0xf3, 0x0f, 0x10, dest.code, src);
2.63 + }
2.64 + void cvttss2si(Register dest, Register src) {
2.65 + assert(Features().sse);
2.66 + emit3(0xf3, 0x0f, 0x2c, dest.code, src.code);
2.67 + }
2.68 + void cvttss2si(Register dest, const Operand &src) {
2.69 + assert(Features().sse);
2.70 + emit3(0xf3, 0x0f, 0x2c, dest.code, src);
2.71 + }
2.72 + void cvtss2si(Register dest, Register src) {
2.73 + assert(Features().sse);
2.74 + emit3(0xf3, 0x0f, 0x2d, dest.code, src.code);
2.75 + }
2.76 + void cvtss2si(Register dest, const Operand &src) {
2.77 + assert(Features().sse);
2.78 + emit3(0xf3, 0x0f, 0x2d, dest.code, src);
2.79 + }
2.80 + void cvtsi2ss(FloatRegister dest, Register src) {
2.81 + assert(Features().sse);
2.82 + emit3(0xf3, 0x0f, 0x2a, dest.code, src.code);
2.83 + }
2.84 + void cvtsi2ss(FloatRegister dest, const Operand &src) {
2.85 + assert(Features().sse);
2.86 + emit3(0xf3, 0x0f, 0x2a, dest.code, src);
2.87 + }
2.88 + void addss(FloatRegister dest, const Operand &src) {
2.89 + assert(Features().sse);
2.90 + emit3(0xf3, 0x0f, 0x58, dest.code, src);
2.91 + }
2.92 + void subss(FloatRegister dest, const Operand &src) {
2.93 + assert(Features().sse);
2.94 + emit3(0xf3, 0x0f, 0x5c, dest.code, src);
2.95 + }
2.96 + void mulss(FloatRegister dest, const Operand &src) {
2.97 + assert(Features().sse);
2.98 + emit3(0xf3, 0x0f, 0x59, dest.code, src);
2.99 + }
2.100 + void divss(FloatRegister dest, const Operand &src) {
2.101 + assert(Features().sse);
2.102 + emit3(0xf3, 0x0f, 0x5e, dest.code, src);
2.103 + }
2.104 + void ucomiss(FloatRegister left, Register right) {
2.105 + emit2(0x0f, 0x2e, left.code, right.code);
2.106 + }
2.107 + void ucomiss(FloatRegister left, const Operand &right) {
2.108 + emit2(0x0f, 0x2e, left.code, right);
2.109 + }
2.110 +
2.111 + // SSE2-only instructions.
2.112 + void movd(Register dest, FloatRegister src) {
2.113 + assert(Features().sse2);
2.114 + emit3(0x66, 0x0f, 0x7e, dest.code, src.code);
2.115 + }
2.116 + void movd(Register dest, const Operand &src) {
2.117 + assert(Features().sse2);
2.118 + emit3(0x66, 0x0f, 0x7e, dest.code, src);
2.119 + }
2.120 +
2.121 static void PatchRel32Absolute(uint8_t *ip, void *ptr) {
2.122 int32_t delta = uint32_t(ptr) - uint32_t(ip);
2.123 *reinterpret_cast<int32_t *>(ip - 4) = delta;
2.124 @@ -806,6 +905,22 @@
2.125 emit(reg, operand);
2.126 }
2.127
2.128 + void emit3(uint8_t prefix1, uint8_t prefix2, uint8_t opcode) {
2.129 + ensureSpace();
2.130 + *pos_++ = prefix1;
2.131 + *pos_++ = prefix2;
2.132 + *pos_++ = opcode;
2.133 + }
2.134 + void emit3(uint8_t prefix1, uint8_t prefix2, uint8_t opcode, uint8_t reg, uint8_t opreg) {
2.135 + emit3(prefix1, prefix2, opcode);
2.136 + assert(reg <= 7);
2.137 + *pos_++ = (kModeReg << 6) | (reg << 3) | opreg;
2.138 + }
2.139 + void emit3(uint8_t prefix1, uint8_t prefix2, uint8_t opcode, uint8_t reg, const Operand &operand) {
2.140 + emit3(prefix1, prefix2, opcode);
2.141 + emit(reg, operand);
2.142 + }
2.143 +
2.144 template <typename T>
2.145 void shift_cl(const T &t, uint8_t r) {
2.146 emit1(0xd3, r, t);
3.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
3.2 +++ b/public/jit/x86/macro-assembler-x86.h Thu Aug 08 20:26:36 2013 -0700
3.3 @@ -0,0 +1,104 @@
3.4 +/**
3.5 + * vim: set ts=8 sts=2 sw=2 tw=99 et:
3.6 + * =============================================================================
3.7 + * SourcePawn JIT SDK
3.8 + * Copyright (C) 2004-2008 AlliedModders LLC. All rights reserved.
3.9 + * =============================================================================
3.10 + *
3.11 + * This program is free software; you can redistribute it and/or modify it under
3.12 + * the terms of the GNU General Public License, version 3.0, as published by the
3.13 + * Free Software Foundation.
3.14 + *
3.15 + * This program is distributed in the hope that it will be useful, but WITHOUT
3.16 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
3.17 + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
3.18 + * details.
3.19 + *
3.20 + * You should have received a copy of the GNU General Public License along with
3.21 + * this program. If not, see <http://www.gnu.org/licenses/>.
3.22 + *
3.23 + * As a special exception, AlliedModders LLC gives you permission to link the
3.24 + * code of this program (as well as its derivative works) to "Half-Life 2," the
3.25 + * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
3.26 + * by the Valve Corporation. You must obey the GNU General Public License in
3.27 + * all respects for all other code used. Additionally, AlliedModders LLC grants
3.28 + * this exception to all derivative works. AlliedModders LLC defines further
3.29 + * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
3.30 + * or <http://www.sourcemod.net/license.php>.
3.31 + *
3.32 + * Version: $Id$
3.33 + */
3.34 +#ifndef _include_sourcepawn_macroassembler_x86h__
3.35 +#define _include_sourcepawn_macroassembler_x86h__
3.36 +
3.37 +#include <assembler.h>
3.38 +#include <ke_vector.h>
3.39 +#include <string.h>
3.40 +#include <assembler-x86.h>
3.41 +
3.42 +class MacroAssemblerX86 : public AssemblerX86
3.43 +{
3.44 + public:
3.45 + static void GenerateFeatureDetection(MacroAssemblerX86 &masm) {
3.46 + masm.push(ebp);
3.47 + masm.movl(ebp, esp);
3.48 + masm.push(ebx);
3.49 + {
3.50 + // Get ECX, EDX feature bits at the first CPUID level.
3.51 + masm.movl(eax, 1);
3.52 + masm.cpuid();
3.53 + masm.movl(eax, Operand(ebp, 8));
3.54 + masm.movl(Operand(eax, 0), ecx);
3.55 + masm.movl(eax, Operand(ebp, 12));
3.56 + masm.movl(Operand(eax, 0), edx);
3.57 + }
3.58 +
3.59 + // Zero out bits we're not guaranteed to get.
3.60 + masm.movl(eax, Operand(ebp, 16));
3.61 + masm.movl(Operand(eax, 0), 0);
3.62 +
3.63 + Label skip_level_7;
3.64 + {
3.65 + // Get EBX feature bits at 7th CPUID level.
3.66 + masm.movl(eax, 0);
3.67 + masm.cpuid();
3.68 + masm.cmpl(eax, 7);
3.69 + masm.j(below, &skip_level_7);
3.70 + masm.movl(eax, 7);
3.71 + masm.movl(ecx, 0);
3.72 + masm.cpuid();
3.73 + masm.movl(eax, Operand(ebp, 16));
3.74 + masm.movl(Operand(eax, 0), ebx);
3.75 + }
3.76 + masm.bind(&skip_level_7);
3.77 +
3.78 + masm.pop(ebx);
3.79 + masm.pop(ebp);
3.80 + masm.ret();
3.81 + }
3.82 +
3.83 + static void RunFeatureDetection(void *code) {
3.84 + typedef void (*fn_t)(int *reg_ecx, int *reg_edx, int *reg_ebx);
3.85 +
3.86 + int reg_ecx, reg_edx, reg_ebx;
3.87 + ((fn_t)code)(®_ecx, ®_edx, ®_ebx);
3.88 +
3.89 + CPUFeatures features;
3.90 + features.fpu = !!(reg_edx & (1 << 0));
3.91 + features.mmx = !!(reg_edx & (1 << 23));
3.92 + features.sse = !!(reg_edx & (1 << 25));
3.93 + features.sse2 = !!(reg_edx & (1 << 26));
3.94 + features.sse3 = !!(reg_ecx & (1 << 0));
3.95 + features.ssse3 = !!(reg_ecx & (1 << 9));
3.96 + features.sse4_1 = !!(reg_ecx & (1 << 19));
3.97 + features.sse4_2 = !!(reg_ecx & (1 << 20));
3.98 + features.avx = !!(reg_ecx & (1 << 28));
3.99 + features.avx2 = !!(reg_ebx & (1 << 5));
3.100 + SetFeatures(features);
3.101 + }
3.102 +
3.103 + private:
3.104 +};
3.105 +
3.106 +#endif // _include_sourcepawn_macroassembler_x86h__
3.107 +
4.1 --- a/sourcepawn/jit/AMBuilder Thu Aug 08 20:21:37 2013 -0700
4.2 +++ b/sourcepawn/jit/AMBuilder Thu Aug 08 20:26:36 2013 -0700
4.3 @@ -37,7 +37,8 @@
4.4 'zlib/uncompr.c',
4.5 'zlib/zutil.c',
4.6 'md5/md5.cpp',
4.7 - '../../knight/shared/KeCodeAllocator.cpp'
4.8 + '../../knight/shared/KeCodeAllocator.cpp',
4.9 + '../../public/jit/x86/assembler-x86.cpp'
4.10 ])
4.11 SM.AutoVersion('sourcepawn/jit', binary)
4.12 SM.ExtractDebugInfo(extension, binary)
5.1 --- a/sourcepawn/jit/Makefile.shell Thu Aug 08 20:21:37 2013 -0700
5.2 +++ b/sourcepawn/jit/Makefile.shell Thu Aug 08 20:26:36 2013 -0700
5.3 @@ -34,6 +34,7 @@
5.4 zlib/zutil.c \
5.5
5.6 OBJECTS += ../../knight/shared/KeCodeAllocator.cpp
5.7 +OBJECTS += ../../public/jit/x86/assembler-x86.cpp
5.8
5.9 ##############################################
5.10 ### CONFIGURE ANY OTHER FLAGS/OPTIONS HERE ###
5.11 @@ -75,6 +76,7 @@
5.12 endif
5.13
5.14 OBJ_LINUX := $(OBJECTS:../../knight/shared/%.cpp=$(BIN_DIR)/knight/%.o)
5.15 +OBJ_LINUX := $(OBJ_LINUX:../../public/jit/x86/%.cpp=$(BIN_DIR)/%.o)
5.16 OBJ_LINUX := $(OBJ_LINUX:%.cpp=$(BIN_DIR)/%.o)
5.17 OBJ_LINUX := $(OBJ_LINUX:%.c=$(BIN_DIR)/%.o)
5.18
5.19 @@ -89,6 +91,9 @@
5.20 $(BIN_DIR)/knight/%.o: ../../knight/shared/%.cpp
5.21 $(CXX) $(INCLUDE) $(CFLAGS) $(CXXFLAGS) -o $@ -c $<
5.22
5.23 +$(BIN_DIR)/assembler-x86.o: ../../public/jit/x86/assembler-x86.cpp
5.24 + $(CXX) $(INCLUDE) $(CFLAGS) $(CXXFLAGS) -o $@ -c $<
5.25 +
5.26 all:
5.27 mkdir -p $(BIN_DIR)/x86
5.28 mkdir -p $(BIN_DIR)/md5
6.1 --- a/sourcepawn/jit/x86/jit_x86.cpp Thu Aug 08 20:21:37 2013 -0700
6.2 +++ b/sourcepawn/jit/x86/jit_x86.cpp Thu Aug 08 20:26:36 2013 -0700
6.3 @@ -983,10 +983,15 @@
6.4 break;
6.5
6.6 case OP_FLOAT:
6.7 - __ fild32(Operand(edi, 0));
6.8 - __ subl(esp, 4);
6.9 - __ fstp32(Operand(esp, 0));
6.10 - __ pop(pri);
6.11 + if (MacroAssemblerX86::Features().sse2) {
6.12 + __ cvtsi2ss(xmm0, Operand(edi, 0));
6.13 + __ movd(pri, xmm0);
6.14 + } else {
6.15 + __ fild32(Operand(edi, 0));
6.16 + __ subl(esp, 4);
6.17 + __ fstp32(Operand(esp, 0));
6.18 + __ pop(pri);
6.19 + }
6.20 __ addl(stk, 4);
6.21 break;
6.22
6.23 @@ -994,34 +999,52 @@
6.24 case OP_FLOATSUB:
6.25 case OP_FLOATMUL:
6.26 case OP_FLOATDIV:
6.27 - __ subl(esp, 4);
6.28 - __ fld32(Operand(edi, 0));
6.29 + if (MacroAssemblerX86::Features().sse2) {
6.30 + __ movss(xmm0, Operand(stk, 0));
6.31 + if (op == OP_FLOATADD)
6.32 + __ addss(xmm0, Operand(stk, 4));
6.33 + else if (op == OP_FLOATSUB)
6.34 + __ subss(xmm0, Operand(stk, 4));
6.35 + else if (op == OP_FLOATMUL)
6.36 + __ mulss(xmm0, Operand(stk, 4));
6.37 + else if (op == OP_FLOATDIV)
6.38 + __ divss(xmm0, Operand(stk, 4));
6.39 + __ movd(pri, xmm0);
6.40 + } else {
6.41 + __ subl(esp, 4);
6.42 + __ fld32(Operand(stk, 0));
6.43
6.44 - if (op == OP_FLOATADD)
6.45 - __ fadd32(Operand(edi, 4));
6.46 - else if (op == OP_FLOATSUB)
6.47 - __ fsub32(Operand(edi, 4));
6.48 - else if (op == OP_FLOATMUL)
6.49 - __ fmul32(Operand(edi, 4));
6.50 - else if (op == OP_FLOATDIV)
6.51 - __ fdiv32(Operand(edi, 4));
6.52 + if (op == OP_FLOATADD)
6.53 + __ fadd32(Operand(stk, 4));
6.54 + else if (op == OP_FLOATSUB)
6.55 + __ fsub32(Operand(stk, 4));
6.56 + else if (op == OP_FLOATMUL)
6.57 + __ fmul32(Operand(stk, 4));
6.58 + else if (op == OP_FLOATDIV)
6.59 + __ fdiv32(Operand(stk, 4));
6.60
6.61 - __ fstp32(Operand(esp, 0));
6.62 - __ pop(pri);
6.63 + __ fstp32(Operand(esp, 0));
6.64 + __ pop(pri);
6.65 + }
6.66 __ addl(stk, 8);
6.67 break;
6.68
6.69 case OP_RND_TO_NEAREST:
6.70 {
6.71 - static float kRoundToNearest = 0.5f;
6.72 - // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf
6.73 - __ fld32(Operand(edi, 0));
6.74 - __ fadd32(st0, st0);
6.75 - __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));
6.76 - __ subl(esp, 4);
6.77 - __ fistp32(Operand(esp, 0));
6.78 - __ pop(pri);
6.79 - __ sarl(pri, 1);
6.80 + if (MacroAssemblerX86::Features().sse) {
6.81 + // Assume no one is touching MXCSR.
6.82 + __ cvtss2si(pri, Operand(stk, 0));
6.83 + } else {
6.84 + static float kRoundToNearest = 0.5f;
6.85 + // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf
6.86 + __ fld32(Operand(stk, 0));
6.87 + __ fadd32(st0, st0);
6.88 + __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));
6.89 + __ subl(esp, 4);
6.90 + __ fistp32(Operand(esp, 0));
6.91 + __ pop(pri);
6.92 + __ sarl(pri, 1);
6.93 + }
6.94 __ addl(stk, 4);
6.95 break;
6.96 }
6.97 @@ -1030,7 +1053,7 @@
6.98 {
6.99 static float kRoundToCeil = -0.5f;
6.100 // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf
6.101 - __ fld32(Operand(edi, 0));
6.102 + __ fld32(Operand(stk, 0));
6.103 __ fadd32(st0, st0);
6.104 __ fsubr32(Operand(ExternalAddress(&kRoundToCeil)));
6.105 __ subl(esp, 4);
6.106 @@ -1043,15 +1066,19 @@
6.107 }
6.108
6.109 case OP_RND_TO_ZERO:
6.110 - __ fld32(Operand(edi, 0));
6.111 - __ subl(esp, 8);
6.112 - __ fstcw(Operand(esp, 4));
6.113 - __ movl(Operand(esp, 0), 0xfff);
6.114 - __ fldcw(Operand(esp, 0));
6.115 - __ fistp32(Operand(esp, 0));
6.116 - __ pop(pri);
6.117 - __ fldcw(Operand(esp, 0));
6.118 - __ addl(esp, 4);
6.119 + if (MacroAssemblerX86::Features().sse) {
6.120 + __ cvttss2si(pri, Operand(stk, 0));
6.121 + } else {
6.122 + __ fld32(Operand(stk, 0));
6.123 + __ subl(esp, 8);
6.124 + __ fstcw(Operand(esp, 4));
6.125 + __ movl(Operand(esp, 0), 0xfff);
6.126 + __ fldcw(Operand(esp, 0));
6.127 + __ fistp32(Operand(esp, 0));
6.128 + __ pop(pri);
6.129 + __ fldcw(Operand(esp, 0));
6.130 + __ addl(esp, 4);
6.131 + }
6.132 __ addl(stk, 4);
6.133 break;
6.134
6.135 @@ -1071,10 +1098,15 @@
6.136 case OP_FLOATCMP:
6.137 {
6.138 Label bl, ab, done;
6.139 - __ fld32(Operand(edi, 0));
6.140 - __ fld32(Operand(edi, 4));
6.141 - __ fucomip(st1);
6.142 - __ fstp(st0);
6.143 + if (MacroAssemblerX86::Features().sse) {
6.144 + __ movss(xmm0, Operand(stk, 4));
6.145 + __ ucomiss(xmm0, Operand(stk, 0));
6.146 + } else {
6.147 + __ fld32(Operand(stk, 0));
6.148 + __ fld32(Operand(stk, 4));
6.149 + __ fucomip(st1);
6.150 + __ fstp(st0);
6.151 + }
6.152 __ j(above, &ab);
6.153 __ j(below, &bl);
6.154 __ xorl(pri, pri);
6.155 @@ -1869,6 +1901,14 @@
6.156 if (!m_pJitGenArray)
6.157 return false;
6.158
6.159 + MacroAssemblerX86 masm;
6.160 + MacroAssemblerX86::GenerateFeatureDetection(masm);
6.161 + void *code = LinkCode(masm);
6.162 + if (!code)
6.163 + return false;
6.164 + MacroAssemblerX86::RunFeatureDetection(code);
6.165 + KE_FreeCode(g_pCodeCache, code);
6.166 +
6.167 return true;
6.168 }
6.169
7.1 --- a/sourcepawn/jit/x86/jit_x86.h Thu Aug 08 20:21:37 2013 -0700
7.2 +++ b/sourcepawn/jit/x86/jit_x86.h Thu Aug 08 20:26:36 2013 -0700
7.3 @@ -35,7 +35,7 @@
7.4 #include <sp_vm_types.h>
7.5 #include <sp_vm_api.h>
7.6 #include <KeCodeAllocator.h>
7.7 -#include <assembler-x86.h>
7.8 +#include <macro-assembler-x86.h>
7.9 #include <ke_vector.h>
7.10 #include "jit_shared.h"
7.11 #include "BaseRuntime.h"