sourcemod-central: sourcepawn/jit/x86/jit

     1 /**

     2  * vim: set ts=2 sw=2 tw=99 et:

     3  * =============================================================================

     4  * SourceMod

     5  * Copyright (C) 2004-2008 AlliedModders LLC.  All rights reserved.

     6  * =============================================================================

     7  *

     8  * This program is free software; you can redistribute it and/or modify it under

     9  * the terms of the GNU General Public License, version 3.0, as published by the

    10  * Free Software Foundation.

    11  *

    12  * This program is distributed in the hope that it will be useful, but WITHOUT

    13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    14  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more

    15  * details.

    16  *

    17  * You should have received a copy of the GNU General Public License along with

    18  * this program.  If not, see <http://www.gnu.org/licenses/>.

    19  *

    20  * As a special exception, AlliedModders LLC gives you permission to link the

    21  * code of this program (as well as its derivative works) to "Half-Life 2," the

    22  * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software

    23  * by the Valve Corporation.  You must obey the GNU General Public License in

    24  * all respects for all other code used.  Additionally, AlliedModders LLC grants

    25  * this exception to all derivative works.  AlliedModders LLC defines further

    26  * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),

    27  * or <http://www.sourcemod.net/license.php>.

    28  *

    29  * Version: $Id$

    30  */

    32 #include <string.h>

    33 #include <stdlib.h>

    34 #include <assert.h>

    35 #include "jit_x86.h"

    36 #include "../sp_vm_engine.h"

    37 #include "../engine2.h"

    38 #include "../BaseRuntime.h"

    39 #include "../sp_vm_basecontext.h"

    41 using namespace Knight;

    43 #if defined USE_UNGEN_OPCODES

    44 #include "ungen_opcodes.h"

    45 #endif

    47 #define __ masm.

    49 JITX86 g_Jit;

    50 KeCodeCache *g_pCodeCache = NULL;

    51 ISourcePawnEngine *engine = &g_engine1;

    53 static inline void *

    54 LinkCode(AssemblerX86 &masm)

    55 {

    56   if (masm.outOfMemory())

    57     return NULL;

    59   void *code = Knight::KE_AllocCode(g_pCodeCache, masm.length());

    60   if (!code)

    61     return NULL;

    63   masm.emitToExecutableMemory(code);

    64   return code;

    65 }

    67 static inline ConditionCode

    68 OpToCondition(OPCODE op)

    69 {

    70   switch (op) {

    71    case OP_EQ:

    72    case OP_JEQ:

    73     return equal;

    74    case OP_NEQ:

    75    case OP_JNEQ:

    76     return not_equal;

    77    case OP_SLESS:

    78    case OP_JSLESS:

    79     return less;

    80    case OP_SLEQ:

    81    case OP_JSLEQ:

    82     return less_equal;

    83    case OP_SGRTR:

    84    case OP_JSGRTR:

    85     return greater;

    86    case OP_SGEQ:

    87    case OP_JSGEQ:

    88     return greater_equal;

    89    default:

    90     assert(false);

    91     return negative;

    92   }

    93 }

    95 struct array_creation_t

    96 {

    97   const cell_t *dim_list;     /* Dimension sizes */

    98   cell_t dim_count;           /* Number of dimensions */

    99   cell_t *data_offs;          /* Current offset AFTER the indirection vectors (data) */

   100   cell_t *base;               /* array base */

   101 };

   103 static cell_t

   104 GenerateInnerArrayIndirectionVectors(array_creation_t *ar, int dim, cell_t cur_offs)

   105 {

   106   cell_t write_offs = cur_offs;

   107   cell_t *data_offs = ar->data_offs;

   109   cur_offs += ar->dim_list[dim];

   111   // Dimension n-x where x > 2 will have sub-vectors.

   112   // Otherwise, we just need to reference the data section.

   113   if (ar->dim_count > 2 && dim < ar->dim_count - 2) {

   114     // For each index at this dimension, write offstes to our sub-vectors.

   115     // After we write one sub-vector, we generate its sub-vectors recursively.

   116     // At the end, we're given the next offset we can use.

   117     for (int i = 0; i < ar->dim_list[dim]; i++) {

   118       ar->base[write_offs] = (cur_offs - write_offs) * sizeof(cell_t);

   119       write_offs++;

   120       cur_offs = GenerateInnerArrayIndirectionVectors(ar, dim + 1, cur_offs);

   121     }

   122   } else {

   123     // In this section, there are no sub-vectors, we need to write offsets

   124     // to the data.  This is separate so the data stays in one big chunk.

   125     // The data offset will increment by the size of the last dimension,

   126     // because that is where the data is finally computed as.

   127     for (int i = 0; i < ar->dim_list[dim]; i++) {

   128       ar->base[write_offs] = (*data_offs - write_offs) * sizeof(cell_t);

   129       write_offs++;

   130       *data_offs = *data_offs + ar->dim_list[dim + 1];

   131     }

   132   }

   134   return cur_offs;

   135 }

   137 static cell_t

   138 calc_indirection(const array_creation_t *ar, cell_t dim)

   139 {

   140   cell_t size = ar->dim_list[dim];

   141   if (dim < ar->dim_count - 2)

   142     size += ar->dim_list[dim] * calc_indirection(ar, dim + 1);

   143   return size;

   144 }

   146 static cell_t

   147 GenerateArrayIndirectionVectors(cell_t *arraybase, cell_t dims[], cell_t _dimcount, bool autozero)

   148 {

   149   array_creation_t ar;

   150   cell_t data_offs;

   152   /* Reverse the dimensions */

   153   cell_t dim_list[sDIMEN_MAX];

   154   int cur_dim = 0;

   155   for (int i = _dimcount - 1; i >= 0; i--)

   156     dim_list[cur_dim++] = dims[i];

   158   ar.base = arraybase;

   159   ar.dim_list = dim_list;

   160   ar.dim_count = _dimcount;

   161   ar.data_offs = &data_offs;

   163   data_offs = calc_indirection(&ar, 0);

   164   GenerateInnerArrayIndirectionVectors(&ar, 0, 0);

   165   return data_offs;

   166 }

   168 static int

   169 PopTrackerAndSetHeap(sp_plugin_t *plugin, InfoVars &vars)

   170 {

   171 	tracker_t *trk = (tracker_t *)(vars.ctx->vm[JITVARS_TRACKER]);

   172   assert(trk->pCur > trk->pBase);

   174   trk->pCur--;

   175   if (trk->pCur < trk->pBase)

   176     return SP_ERROR_TRACKER_BOUNDS;

   178   ucell_t amt = *trk->pCur;

   179   if (amt > (vars.hp - plugin->data_size))

   180     return SP_ERROR_HEAPMIN;

   182   vars.hp -= amt;

   183   return SP_ERROR_NONE;

   184 }

   186 static int

   187 PushTracker(sp_context_t *ctx, size_t amount)

   188 {

   189   tracker_t *trk = (tracker_t *)(ctx->vm[JITVARS_TRACKER]);

   191   if ((size_t)(trk->pCur - trk->pBase) >= trk->size)

   192     return SP_ERROR_TRACKER_BOUNDS;

   194   if (trk->pCur + 1 - (trk->pBase + trk->size) == 0) {

   195     size_t disp = trk->size - 1;

   196     trk->size *= 2;

   197     trk->pBase = (ucell_t *)realloc(trk->pBase, trk->size * sizeof(cell_t));

   199     if (!trk->pBase)

   200       return SP_ERROR_TRACKER_BOUNDS;

   202     trk->pCur = trk->pBase + disp;

   203   }

   205   *trk->pCur++ = amount;

   206   return SP_ERROR_NONE;

   207 }

   209 static int

   210 GenerateArray(sp_plugin_t *plugin, InfoVars &vars, uint32_t argc, cell_t *argv, int autozero)

   211 {

   212   // Calculate how many cells are needed.

   213   if (argv[0] <= 0)

   214     return SP_ERROR_ARRAY_TOO_BIG;

   216   uint32_t dim = 1;         // second to last dimension

   217   uint32_t cells = argv[0];

   219   for (uint32_t dim = 1; dim < argc; dim++) {

   220     cell_t dimsize = argv[dim];

   221     if (dimsize <= 0)

   222       return SP_ERROR_ARRAY_TOO_BIG;

   223     if (!ke::IsUint32MultiplySafe(cells, dimsize))

   224       return SP_ERROR_ARRAY_TOO_BIG;

   225     cells *= uint32_t(dimsize);

   226     if (!ke::IsUint32AddSafe(cells, dimsize))

   227       return SP_ERROR_ARRAY_TOO_BIG;

   228     cells += uint32_t(dimsize);

   229   }

   231   if (!ke::IsUint32MultiplySafe(cells, 4))

   232     return SP_ERROR_ARRAY_TOO_BIG;

   234   uint32_t bytes = cells * 4;

   235   if (!ke::IsUint32AddSafe(vars.hp, bytes))

   236     return SP_ERROR_ARRAY_TOO_BIG;

   238   uint32_t new_hp = vars.hp + bytes;

   239   cell_t *dat_hp = reinterpret_cast<cell_t *>(plugin->memory + new_hp);

   241   // argv, coincidentally, is STK.

   242   if (dat_hp >= argv - STACK_MARGIN)

   243     return SP_ERROR_HEAPLOW;

   245   if (int err = PushTracker(vars.ctx, bytes))

   246     return err;

   248   cell_t *base = reinterpret_cast<cell_t *>(plugin->memory + vars.hp);

   249   cell_t offs = GenerateArrayIndirectionVectors(base, argv, argc, autozero);

   250   assert(size_t(offs) == cells);

   252   argv[argc - 1] = vars.hp;

   253   vars.hp = new_hp;

   254   return SP_ERROR_NONE;

   255 }

   257 #if !defined NDEBUG

   258 static const char *

   259 GetFunctionName(const sp_plugin_t *plugin, uint32_t offs)

   260 {

   261   if (!plugin->debug.unpacked) {

   262     uint32_t max, iter;

   263     sp_fdbg_symbol_t *sym;

   264     sp_fdbg_arraydim_t *arr;

   265     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   267     max = plugin->debug.syms_num;

   268     for (iter = 0; iter < max; iter++) {

   269       sym = (sp_fdbg_symbol_t *)cursor;

   271       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   272         return plugin->debug.stringbase + sym->name;

   274       if (sym->dimcount > 0) {

   275         cursor += sizeof(sp_fdbg_symbol_t);

   276         arr = (sp_fdbg_arraydim_t *)cursor;

   277         cursor += sizeof(sp_fdbg_arraydim_t) * sym->dimcount;

   278         continue;

   279       }

   281       cursor += sizeof(sp_fdbg_symbol_t);

   282     }

   283   } else {

   284     uint32_t max, iter;

   285     sp_u_fdbg_symbol_t *sym;

   286     sp_u_fdbg_arraydim_t *arr;

   287     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   289     max = plugin->debug.syms_num;

   290     for (iter = 0; iter < max; iter++) {

   291       sym = (sp_u_fdbg_symbol_t *)cursor;

   293       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   294         return plugin->debug.stringbase + sym->name;

   296       if (sym->dimcount > 0) {

   297         cursor += sizeof(sp_u_fdbg_symbol_t);

   298         arr = (sp_u_fdbg_arraydim_t *)cursor;

   299         cursor += sizeof(sp_u_fdbg_arraydim_t) * sym->dimcount;

   300         continue;

   301       }

   303       cursor += sizeof(sp_u_fdbg_symbol_t);

   304     }

   305   }

   307   return NULL;

   308 }

   309 #endif

   311 static int

   312 CompileFromThunk(BaseRuntime *runtime, cell_t pcode_offs, void **addrp, char *pc)

   313 {

   314   JitFunction *fn = runtime->GetJittedFunctionByOffset(pcode_offs);

   315   if (!fn) {

   316     int err;

   317     fn = g_Jit.CompileFunction(runtime, pcode_offs, &err);

   318     if (!fn)

   319       return err;

   320   }

   322 #if !defined NDEBUG

   323   g_engine1.GetDebugHook()->OnDebugSpew(

   324       "Patching thunk to %s::%s\n",

   325       runtime->plugin()->name,

   326       GetFunctionName(runtime->plugin(), pcode_offs));

   327 #endif

   329   *addrp = fn->GetEntryAddress();

   331   /* Right now, we always keep the code RWE */

   332   *(intptr_t *)(pc - 4) = intptr_t(fn->GetEntryAddress()) - intptr_t(pc);

   333   return SP_ERROR_NONE;

   334 }

   336 static cell_t

   337 NativeCallback(sp_context_t *ctx, ucell_t native_idx, cell_t *params)

   338 {

   339   sp_native_t *native;

   340   cell_t save_sp = ctx->sp;

   341   cell_t save_hp = ctx->hp;

   342   sp_plugin_t *pl = (sp_plugin_t *)(ctx->vm[JITVARS_PLUGIN]);

   344   ctx->n_idx = native_idx;

   346   if (ctx->hp < (cell_t)pl->data_size) {

   347     ctx->n_err = SP_ERROR_HEAPMIN;

   348     return 0;

   349   }

   351   if (ctx->hp + STACK_MARGIN > ctx->sp) {

   352     ctx->n_err = SP_ERROR_STACKLOW;

   353     return 0;

   354   }

   356   if ((uint32_t)ctx->sp >= pl->mem_size) {

   357     ctx->n_err = SP_ERROR_STACKMIN;

   358     return 0;

   359   }

   361   native = &pl->natives[native_idx];

   363   if (native->status == SP_NATIVE_UNBOUND) {

   364     ctx->n_err = SP_ERROR_INVALID_NATIVE;

   365     return 0;

   366   }

   368   cell_t result = native->pfn(GET_CONTEXT(ctx), params);

   370   if (ctx->n_err != SP_ERROR_NONE)

   371     return result;

   373   if (save_sp != ctx->sp) {

   374     ctx->n_err = SP_ERROR_STACKLEAK;

   375     return result;

   376   }

   377   if (save_hp != ctx->hp) {

   378     ctx->n_err = SP_ERROR_HEAPLEAK;

   379     return result;

   380   }

   382   return result;

   383 }

   385 Compiler::Compiler(BaseRuntime *rt, cell_t pcode_offs)

   386   : rt_(rt),

   387     plugin_(rt->plugin()),

   388     error_(SP_ERROR_NONE),

   389     pcode_start_(pcode_offs),

   390     code_start_(reinterpret_cast<cell_t *>(plugin_->pcode + pcode_start_)),

   391     cip_(code_start_),

   392     code_end_(reinterpret_cast<cell_t *>(plugin_->pcode + plugin_->pcode_size))

   393 {

   394   size_t nmaxops = plugin_->pcode_size / sizeof(cell_t) + 1;

   395   jump_map_ = new Label[nmaxops];

   396 }

   398 Compiler::~Compiler()

   399 {

   400   delete [] jump_map_;

   401 }

   403 JitFunction *

   404 Compiler::emit(int *errp)

   405 {

   406   if (cip_ >= code_end_ || *cip_ != OP_PROC) {

   407     *errp = SP_ERROR_INVALID_INSTRUCTION;

   408     return NULL;

   409   }

   411 #if !defined NDEBUG

   412   g_engine1.GetDebugHook()->OnDebugSpew(

   413       "Compiling function %s::%s\n",

   414       plugin_->name,

   415       GetFunctionName(plugin_, pcode_start_));

   417   SpewOpcode(plugin_, code_start_, cip_);

   418 #endif

   420   cell_t *codeseg = reinterpret_cast<cell_t *>(plugin_->pcode);

   422   cip_++;

   423   if (!emitOp(OP_PROC)) {

   424       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   425       return NULL;

   426   }

   428   while (cip_ < code_end_) {

   429     // If we reach the end of this function, or the beginning of a new

   430     // procedure, then stop.

   431     if (*cip_ == OP_PROC || *cip_ == OP_ENDPROC)

   432       break;

   434 #if !defined NDEBUG

   435     SpewOpcode(plugin_, code_start_, cip_);

   436 #endif

   438     // We assume every instruction is a jump target, so before emitting

   439     // an opcode, we bind its corresponding label.

   440     __ bind(&jump_map_[cip_ - codeseg]);

   442     OPCODE op = (OPCODE)readCell();

   443     if (!emitOp(op) || error_ != SP_ERROR_NONE) {

   444       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   445       return NULL;

   446     }

   447   }

   449   emitCallThunks();

   450   emitErrorPaths();

   452   void *code = LinkCode(masm);

   453   if (!code) {

   454     *errp = SP_ERROR_OUT_OF_MEMORY;

   455     return NULL;

   456   }

   458   return new JitFunction(code, pcode_start_);

   459 }

   461 bool

   462 Compiler::emitOp(OPCODE op)

   463 {

   464   switch (op) {

   465     case OP_MOVE_PRI:

   466       __ movl(pri, alt);

   467       break;

   469     case OP_MOVE_ALT:

   470       __ movl(alt, pri);

   471       break;

   473     case OP_XCHG:

   474       __ xchgl(pri, alt);

   475       break;

   477     case OP_ZERO:

   478     {

   479       cell_t offset = readCell();

   480       __ movl(Operand(dat, offset), 0);

   481       break;

   482     }

   484     case OP_ZERO_S:

   485     {

   486       cell_t offset = readCell();

   487       __ movl(Operand(frm, offset), 0);

   488       break;

   489     }

   491     case OP_PUSH_PRI:

   492     case OP_PUSH_ALT:

   493     {

   494       Register reg = (op == OP_PUSH_PRI) ? pri : alt;

   495       __ movl(Operand(stk, -4), reg);

   496       __ subl(stk, 4);

   497       break;

   498     }

   500     case OP_PUSH_C:

   501     case OP_PUSH2_C:

   502     case OP_PUSH3_C:

   503     case OP_PUSH4_C:

   504     case OP_PUSH5_C:

   505     {

   506       int n = 1;

   507       if (op >= OP_PUSH2_C)

   508         n = ((op - OP_PUSH2_C) / 4) + 2;

   510       int i = 1;

   511       do {

   512         cell_t val = readCell();

   513         __ movl(Operand(stk, -(4 * i)), val);

   514       } while (i++ < n);

   515       __ subl(stk, 4 * n);

   516       break;

   517     }

   519     case OP_PUSH_ADR:

   520     case OP_PUSH2_ADR:

   521     case OP_PUSH3_ADR:

   522     case OP_PUSH4_ADR:

   523     case OP_PUSH5_ADR:

   524     {

   525       int n = 1;

   526       if (op >= OP_PUSH2_ADR)

   527         n = ((op - OP_PUSH2_ADR) / 4) + 2;

   529       int i = 1;

   531       // We temporarily relocate FRM to be a local address instead of an

   532       // absolute address.

   533       __ subl(frm, dat);

   534       do {

   535         cell_t offset = readCell();

   536         __ lea(tmp, Operand(frm, offset));

   537         __ movl(Operand(stk, -(4 * i)), tmp);

   538       } while (i++ < n);

   539       __ subl(stk, 4 * n);

   540       __ addl(frm, dat);

   541       break;

   542     }

   544     case OP_PUSH_S:

   545     case OP_PUSH2_S:

   546     case OP_PUSH3_S:

   547     case OP_PUSH4_S:

   548     case OP_PUSH5_S:

   549     {

   550       int n = 1;

   551       if (op >= OP_PUSH2_S)

   552         n = ((op - OP_PUSH2_S) / 4) + 2;

   554       int i = 1;

   555       do {

   556         cell_t offset = readCell();

   557         __ movl(tmp, Operand(frm, offset));

   558         __ movl(Operand(stk, -(4 * i)), tmp);

   559       } while (i++ < n);

   560       __ subl(stk, 4 * n);

   561       break;

   562     }

   564     case OP_PUSH:

   565     case OP_PUSH2:

   566     case OP_PUSH3:

   567     case OP_PUSH4:

   568     case OP_PUSH5:

   569     {

   570       int n = 1;

   571       if (op >= OP_PUSH2)

   572         n = ((op - OP_PUSH2) / 4) + 2;

   574       int i = 1;

   575       do {

   576         cell_t offset = readCell();

   577         __ movl(tmp, Operand(dat, offset));

   578         __ movl(Operand(stk, -(4 * i)), tmp);

   579       } while (i++ < n);

   580       __ subl(stk, 4 * n);

   581       break;

   582     }

   584     case OP_ZERO_PRI:

   585       __ xorl(pri, pri);

   586       break;

   588     case OP_ZERO_ALT:

   589       __ xorl(alt, alt);

   590       break;

   592     case OP_ADD:

   593       __ addl(pri, alt);

   594       break;

   596     case OP_SUB:

   597       __ subl(pri, alt);

   598       break;

   600     case OP_SUB_ALT:

   601       __ movl(tmp, alt);

   602       __ subl(tmp, pri);

   603       __ movl(pri, tmp);

   604       break;

   606     case OP_PROC:

   607       // Push the old frame onto the stack.

   608       __ movl(tmp, Operand(info, AMX_INFO_FRAME));

   609       __ movl(Operand(stk, -4), tmp);

   610       __ subl(stk, 8);    // extra unused slot for non-existant CIP

   612       // Get and store the new frame.

   613       __ movl(tmp, stk);

   614       __ movl(frm, stk);

   615       __ subl(tmp, dat);

   616       __ movl(Operand(info, AMX_INFO_FRAME), tmp);

   618       // Align the stack to 16-bytes (each call adds 4 bytes).

   619       __ subl(esp, 12);

   620       break;

   622     case OP_IDXADDR_B:

   623     {

   624       cell_t val = readCell();

   625       __ shll(pri, val);

   626       __ addl(pri, alt);

   627       break;

   628     }

   630     case OP_SHL:

   631       __ movl(ecx, alt);

   632       __ shll_cl(pri);

   633       break;

   635     case OP_SHR:

   636       __ movl(ecx, alt);

   637       __ shrl_cl(pri);

   638       break;

   640     case OP_SSHR:

   641       __ movl(ecx, alt);

   642       __ sarl_cl(pri);

   643       break;

   645     case OP_SHL_C_PRI:

   646     case OP_SHL_C_ALT:

   647     {

   648       Register reg = (op == OP_SHL_C_PRI) ? pri : alt;

   649       cell_t val = readCell();

   650       __ shll(reg, val);

   651       break;

   652     }

   654     case OP_SHR_C_PRI:

   655     case OP_SHR_C_ALT:

   656     {

   657       Register reg = (op == OP_SHR_C_PRI) ? pri : alt;

   658       cell_t val = readCell();

   659       __ shrl(reg, val);

   660       break;

   661     }

   663     case OP_SMUL:

   664       __ imull(pri, alt);

   665       break;

   667     case OP_NOT:

   668       __ testl(eax, eax);

   669       __ movl(eax, 0);

   670       __ set(zero, r8_al);

   671       break;

   673     case OP_NEG:

   674       __ negl(eax);

   675       break;

   677     case OP_XOR:

   678       __ xorl(pri, alt);

   679       break;

   681     case OP_OR:

   682       __ orl(pri, alt);

   683       break;

   685     case OP_AND:

   686       __ andl(pri, alt);

   687       break;

   689     case OP_INVERT:

   690       __ notl(pri);

   691       break;

   693     case OP_ADD_C:

   694     {

   695       cell_t val = readCell();

   696       __ addl(pri, val);

   697       break;

   698     }

   700     case OP_SMUL_C:

   701     {

   702       cell_t val = readCell();

   703       __ imull(pri, pri, val);

   704       break;

   705     }

   707     case OP_EQ:

   708     case OP_NEQ:

   709     case OP_SLESS:

   710     case OP_SLEQ:

   711     case OP_SGRTR:

   712     case OP_SGEQ:

   713     {

   714       ConditionCode cc = OpToCondition(op);

   715       __ cmpl(pri, alt);

   716       __ movl(pri, 0);

   717       __ set(cc, r8_al);

   718       break;

   719     }

   721     case OP_EQ_C_PRI:

   722     case OP_EQ_C_ALT:

   723     {

   724       Register reg = (op == OP_EQ_C_PRI) ? pri : alt;

   725       cell_t val = readCell();

   726       __ cmpl(reg, val);

   727       __ movl(pri, 0);

   728       __ set(equal, r8_al);

   729       break;

   730     }

   732     case OP_INC_PRI:

   733     case OP_INC_ALT:

   734     {

   735       Register reg = (OP_INC_PRI) ? pri : alt;

   736       __ addl(reg, 1);

   737       break;

   738     }

   740     case OP_INC:

   741     case OP_INC_S:

   742     {

   743       Register base = (op == OP_INC) ? dat : frm;

   744       cell_t offset = readCell();

   745       __ addl(Operand(base, offset), 1);

   746       break;

   747     }

   749     case OP_INC_I:

   750       __ addl(Operand(dat, pri, NoScale), 1);

   751       break;

   753     case OP_DEC_PRI:

   754     case OP_DEC_ALT:

   755     {

   756       Register reg = (op == OP_DEC_PRI) ? pri : alt;

   757       __ subl(reg, 1);

   758       break;

   759     }

   761     case OP_DEC:

   762     case OP_DEC_S:

   763     {

   764       Register base = (op == OP_DEC) ? dat : frm;

   765       cell_t offset = readCell();

   766       __ subl(Operand(base, offset), 1);

   767       break;

   768     }

   770     case OP_DEC_I:

   771       __ subl(Operand(dat, pri, NoScale), 1);

   772       break;

   774     case OP_LOAD_PRI:

   775     case OP_LOAD_ALT:

   776     {

   777       Register reg = (op == OP_LOAD_PRI) ? pri : alt;

   778       cell_t offset = readCell();

   779       __ movl(reg, Operand(dat, offset));

   780       break;

   781     }

   783     case OP_LOAD_BOTH:

   784     {

   785       cell_t offs1 = readCell();

   786       cell_t offs2 = readCell();

   787       __ movl(pri, Operand(dat, offs1));

   788       __ movl(alt, Operand(dat, offs2));

   789       break;

   790     }

   792     case OP_LOAD_S_PRI:

   793     case OP_LOAD_S_ALT:

   794     {

   795       Register reg = (op == OP_LOAD_S_PRI) ? pri : alt;

   796       cell_t offset = readCell();

   797       __ movl(reg, Operand(frm, offset));

   798       break;

   799     }

   801     case OP_LOAD_S_BOTH:

   802     {

   803       cell_t offs1 = readCell();

   804       cell_t offs2 = readCell();

   805       __ movl(pri, Operand(frm, offs1));

   806       __ movl(alt, Operand(frm, offs2));

   807       break;

   808     }

   810     case OP_LREF_S_PRI:

   811     case OP_LREF_S_ALT:

   812     {

   813       Register reg = (op == OP_LREF_S_PRI) ? pri : alt;

   814       cell_t offset = readCell();

   815       __ movl(reg, Operand(frm, offset));

   816       __ movl(reg, Operand(dat, reg, NoScale));

   817       break;

   818     }

   820     case OP_CONST_PRI:

   821     case OP_CONST_ALT:

   822     {

   823       Register reg = (op == OP_CONST_PRI) ? pri : alt;

   824       cell_t val = readCell();

   825       __ movl(reg, val);

   826       break;

   827     }

   829     case OP_ADDR_PRI:

   830     case OP_ADDR_ALT:

   831     {

   832       Register reg = (op == OP_ADDR_PRI) ? pri : alt;

   833       cell_t offset = readCell();

   834       __ movl(reg, Operand(info, AMX_INFO_FRAME));

   835       __ addl(reg, offset);

   836       break;

   837     }

   839     case OP_STOR_PRI:

   840     case OP_STOR_ALT:

   841     {

   842       Register reg = (op == OP_STOR_PRI) ? pri : alt;

   843       cell_t offset = readCell();

   844       __ movl(Operand(dat, offset), reg);

   845       break;

   846     }

   848     case OP_STOR_S_PRI:

   849     case OP_STOR_S_ALT:

   850     {

   851       Register reg = (op == OP_STOR_S_PRI) ? pri : alt;

   852       cell_t offset = readCell();

   853       __ movl(Operand(frm, offset), reg);

   854       break;

   855     }

   857     case OP_IDXADDR:

   858       __ lea(pri, Operand(alt, pri, ScaleFour));

   859       break;

   861     case OP_SREF_S_PRI:

   862     case OP_SREF_S_ALT:

   863     {

   864       Register reg = (op == OP_SREF_S_PRI) ? pri : alt;

   865       cell_t offset = readCell();

   866       __ movl(tmp, Operand(frm, offset));

   867       __ movl(Operand(dat, tmp, NoScale), reg);

   868       break;

   869     }

   871     case OP_POP_PRI:

   872     case OP_POP_ALT:

   873     {

   874       Register reg = (op == OP_POP_PRI) ? pri : alt;

   875       __ movl(reg, Operand(stk, 0));

   876       __ addl(stk, 4);

   877       break;

   878     }

   880     case OP_SWAP_PRI:

   881     case OP_SWAP_ALT:

   882     {

   883       Register reg = (op == OP_SWAP_PRI) ? pri : alt;

   884       __ movl(tmp, Operand(stk, 0));

   885       __ movl(Operand(stk, 0), reg);

   886       __ movl(reg, tmp);

   887       break;

   888     }

   890     case OP_LIDX:

   891       __ lea(pri, Operand(alt, pri, ScaleFour));

   892       __ movl(pri, Operand(dat, pri, NoScale));

   893       break;

   895     case OP_LIDX_B:

   896     {

   897       cell_t val = readCell();

   898       if (val >= 0 && val <= 3) {

   899         __ lea(pri, Operand(alt, pri, Scale(val)));

   900       } else {

   901         __ shll(pri, val);

   902         __ addl(pri, alt);

   903       }

   904       emitCheckAddress(pri);

   905       __ movl(pri, Operand(dat, pri, NoScale));

   906       break;

   907     }

   909     case OP_CONST:

   910     case OP_CONST_S:

   911     {

   912       Register base = (op == OP_CONST) ? dat : frm;

   913       cell_t offset = readCell();

   914       cell_t val = readCell();

   915       __ movl(Operand(base, offset), val);

   916       break;

   917     }

   919     case OP_LOAD_I:

   920       emitCheckAddress(pri);

   921       __ movl(pri, Operand(dat, pri, NoScale));

   922       break;

   924     case OP_STOR_I:

   925       emitCheckAddress(alt);

   926       __ movl(Operand(dat, alt, NoScale), pri);

   927       break;

   929     case OP_SDIV:

   930     case OP_SDIV_ALT:

   931     {

   932       Register dividend = (op == OP_SDIV) ? pri : alt;

   933       Register divisor = (op == OP_SDIV) ? alt : pri;

   935       // Guard against divide-by-zero.

   936       __ testl(divisor, divisor);

   937       __ j(zero, &error_divide_by_zero_);

   939       // A more subtle case; -INT_MIN / -1 yields an overflow exception.

   940       Label ok;

   941       __ cmpl(divisor, -1);

   942       __ j(not_equal, &ok);

   943       __ cmpl(dividend, 0x80000000);

   944       __ j(equal, &error_integer_overflow_);

   945       __ bind(&ok);

   947       // Now we can actually perform the divide.

   948       __ movl(tmp, divisor);

   949       if (op == OP_SDIV)

   950         __ movl(edx, dividend);

   951       else

   952         __ movl(eax, dividend);

   953       __ sarl(edx, 31);

   954       __ idivl(tmp);

   955       break;

   956     }

   958     case OP_LODB_I:

   959     {

   960       cell_t val = readCell();

   961       emitCheckAddress(pri);

   962       __ movl(pri, Operand(dat, pri, NoScale));

   963       if (val == 1)

   964         __ andl(pri, 0xff);

   965       else if (val == 2)

   966         __ andl(pri, 0xffff);

   967       break;

   968     }

   970     case OP_STRB_I:

   971     {

   972       cell_t val = readCell();

   973       emitCheckAddress(alt);

   974       if (val == 1)

   975         __ movb(Operand(dat, alt, NoScale), pri);

   976       else if (val == 2)

   977         __ movw(Operand(dat, alt, NoScale), pri);

   978       else if (val == 4)

   979         __ movl(Operand(dat, alt, NoScale), pri);

   980       break;

   981     }

   983     case OP_RETN:

   984     {

   985       // Restore the old frame pointer.

   986       __ movl(frm, Operand(stk, 4));              // get the old frm

   987       __ addl(stk, 8);                            // pop stack

   988       __ movl(Operand(info, AMX_INFO_FRAME), frm);  // store back old frm

   989       __ addl(frm, dat);                          // relocate

   991       // Remove parameters.

   992       __ movl(tmp, Operand(stk, 0));

   993       __ lea(stk, Operand(stk, tmp, ScaleFour, 4));

   994       __ addl(esp, 12);

   995       __ ret();

   996       break;

   997     }

   999     case OP_MOVS:

  1000     {

  1001       cell_t val = readCell();

  1002       unsigned dwords = val / 4;

  1003       unsigned bytes = val % 4;

  1005       __ cld();

  1006       __ push(esi);

  1007       __ push(edi);

  1008       __ lea(esi, Operand(dat, pri, NoScale));

  1009       __ lea(edi, Operand(dat, alt, NoScale));

  1010       if (dwords) {

  1011         __ movl(ecx, dwords);

  1012         __ rep_movsd();

  1013       }

  1014       if (bytes) {

  1015         __ movl(ecx, bytes);

  1016         __ rep_movsb();

  1017       }

  1018       __ pop(edi);

  1019       __ pop(esi);

  1020       break;

  1021     }

  1023     case OP_FILL:

  1024     {

  1025       // eax/pri is used implicitly.

  1026       unsigned dwords = readCell() / 4;

  1027       __ push(edi);

  1028       __ lea(edi, Operand(dat, alt, NoScale));

  1029       __ movl(ecx, dwords);

  1030       __ cld();

  1031       __ rep_stosd();

  1032       __ pop(edi);

  1033       break;

  1034     }

  1036     case OP_STRADJUST_PRI:

  1037       __ addl(pri, 4);

  1038       __ sarl(pri, 2);

  1039       break;

  1041     case OP_FABS:

  1042       __ movl(pri, Operand(stk, 0));

  1043       __ andl(pri, 0x7fffffff);

  1044       __ addl(stk, 4);

  1045       break;

  1047     case OP_FLOAT:

  1048       if (MacroAssemblerX86::Features().sse2) {

  1049         __ cvtsi2ss(xmm0, Operand(edi, 0));

  1050         __ movd(pri, xmm0);

  1051       } else {

  1052         __ fild32(Operand(edi, 0));

  1053         __ subl(esp, 4);

  1054         __ fstp32(Operand(esp, 0));

  1055         __ pop(pri);

  1056       }

  1057       __ addl(stk, 4);

  1058       break;

  1060     case OP_FLOATADD:

  1061     case OP_FLOATSUB:

  1062     case OP_FLOATMUL:

  1063     case OP_FLOATDIV:

  1064       if (MacroAssemblerX86::Features().sse2) {

  1065         __ movss(xmm0, Operand(stk, 0));

  1066         if (op == OP_FLOATADD)

  1067           __ addss(xmm0, Operand(stk, 4));

  1068         else if (op == OP_FLOATSUB)

  1069           __ subss(xmm0, Operand(stk, 4));

  1070         else if (op == OP_FLOATMUL)

  1071           __ mulss(xmm0, Operand(stk, 4));

  1072         else if (op == OP_FLOATDIV)

  1073           __ divss(xmm0, Operand(stk, 4));

  1074         __ movd(pri, xmm0);

  1075       } else {

  1076         __ subl(esp, 4);

  1077         __ fld32(Operand(stk, 0));

  1079         if (op == OP_FLOATADD)

  1080           __ fadd32(Operand(stk, 4));

  1081         else if (op == OP_FLOATSUB)

  1082           __ fsub32(Operand(stk, 4));

  1083         else if (op == OP_FLOATMUL)

  1084           __ fmul32(Operand(stk, 4));

  1085         else if (op == OP_FLOATDIV)

  1086           __ fdiv32(Operand(stk, 4));

  1088         __ fstp32(Operand(esp, 0));

  1089         __ pop(pri);

  1090       }

  1091       __ addl(stk, 8);

  1092       break;

  1094     case OP_RND_TO_NEAREST:

  1095     {

  1096       if (MacroAssemblerX86::Features().sse) {

  1097         // Assume no one is touching MXCSR.

  1098         __ cvtss2si(pri, Operand(stk, 0));

  1099       } else {

  1100         static float kRoundToNearest = 0.5f;

  1101         // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1102         __ fld32(Operand(stk, 0));

  1103         __ fadd32(st0, st0);

  1104         __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));

  1105         __ subl(esp, 4);

  1106         __ fistp32(Operand(esp, 0));

  1107         __ pop(pri);

  1108         __ sarl(pri, 1);

  1109       }

  1110       __ addl(stk, 4);

  1111       break;

  1112     }

  1114     case OP_RND_TO_CEIL:

  1115     {

  1116       static float kRoundToCeil = -0.5f;

  1117       // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1118       __ fld32(Operand(stk, 0));

  1119       __ fadd32(st0, st0);

  1120       __ fsubr32(Operand(ExternalAddress(&kRoundToCeil)));

  1121       __ subl(esp, 4);

  1122       __ fistp32(Operand(esp, 0));

  1123       __ pop(pri);

  1124       __ sarl(pri, 1);

  1125       __ negl(pri);

  1126       __ addl(stk, 4);

  1127       break;

  1128     }

  1130     case OP_RND_TO_ZERO:

  1131       if (MacroAssemblerX86::Features().sse) {

  1132         __ cvttss2si(pri, Operand(stk, 0));

  1133       } else {

  1134         __ fld32(Operand(stk, 0));

  1135         __ subl(esp, 8);

  1136         __ fstcw(Operand(esp, 4));

  1137         __ movl(Operand(esp, 0), 0xfff);

  1138         __ fldcw(Operand(esp, 0));

  1139         __ fistp32(Operand(esp, 0));

  1140         __ pop(pri);

  1141         __ fldcw(Operand(esp, 0));

  1142         __ addl(esp, 4);

  1143       }

  1144       __ addl(stk, 4);

  1145       break;

  1147     case OP_RND_TO_FLOOR:

  1148       __ fld32(Operand(stk, 0));

  1149       __ subl(esp, 8);

  1150       __ fstcw(Operand(esp, 4));

  1151       __ movl(Operand(esp, 0), 0x7ff);

  1152       __ fldcw(Operand(esp, 0));

  1153       __ fistp32(Operand(esp, 0));

  1154       __ pop(eax);

  1155       __ fldcw(Operand(esp, 0));

  1156       __ addl(esp, 4);

  1157       __ addl(stk, 4);

  1158       break;

  1160     case OP_FLOATCMP:

  1161     {

  1162       Label bl, ab, done;

  1163       if (MacroAssemblerX86::Features().sse) {

  1164         __ movss(xmm0, Operand(stk, 4));

  1165         __ ucomiss(xmm0, Operand(stk, 0));

  1166       } else {

  1167         __ fld32(Operand(stk, 0));

  1168         __ fld32(Operand(stk, 4));

  1169         __ fucomip(st1);

  1170         __ fstp(st0);

  1171       }

  1172       __ j(above, &ab);

  1173       __ j(below, &bl);

  1174       __ xorl(pri, pri);

  1175       __ jmp(&done);

  1176       __ bind(&ab);

  1177       __ movl(pri, -1);

  1178       __ jmp(&done);

  1179       __ bind(&bl);

  1180       __ movl(pri, 1);

  1181       __ bind(&done);

  1182       __ addl(stk, 8);

  1183       break;

  1184     }

  1186     case OP_STACK:

  1187     {

  1188       cell_t amount = readCell();

  1189       __ addl(stk, amount);

  1191      if (amount > 0) {

  1192        // Check if the stack went beyond the stack top - usually a compiler error.

  1193        __ cmpl(stk, intptr_t(plugin_->memory + plugin_->mem_size));

  1194        __ j(not_below, &error_stack_min_);

  1195      } else {

  1196        // Check if the stack is going to collide with the heap.

  1197        __ movl(tmp, Operand(info, AMX_INFO_HEAP));

  1198        __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1199        __ cmpl(stk, tmp);

  1200        __ j(below, &error_stack_low_);

  1201      }

  1202      break;

  1203     }

  1205     case OP_HEAP:

  1206     {

  1207       cell_t amount = readCell();

  1208       __ movl(alt, Operand(info, AMX_INFO_HEAP));

  1209       __ addl(Operand(info, AMX_INFO_HEAP), amount);

  1211       if (amount > 0) {

  1212         __ cmpl(Operand(info, AMX_INFO_HEAP), plugin_->data_size);

  1213         __ j(below, &error_heap_min_);

  1214       } else {

  1215         __ movl(tmp, Operand(info, AMX_INFO_HEAP));

  1216         __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1217         __ cmpl(tmp, stk);

  1218         __ j(above, &error_heap_low_);

  1219       }

  1220       break;

  1221     }

  1223     case OP_JUMP:

  1224     {

  1225       Label *target = labelAt(readCell());

  1226       if (!target)

  1227         return false;

  1228       __ jmp(target);

  1229       break;

  1230     }

  1232     case OP_JZER:

  1233     case OP_JNZ:

  1234     {

  1235       ConditionCode cc = (op == OP_JZER) ? zero : not_zero;

  1236       Label *target = labelAt(readCell());

  1237       if (!target)

  1238         return false;

  1239       __ testl(pri, pri);

  1240       __ j(cc, target);

  1241       break;

  1242     }

  1244     case OP_JEQ:

  1245     case OP_JNEQ:

  1246     case OP_JSLESS:

  1247     case OP_JSLEQ:

  1248     case OP_JSGRTR:

  1249     case OP_JSGEQ:

  1250     {

  1251       Label *target = labelAt(readCell());

  1252       if (!target)

  1253         return false;

  1254       ConditionCode cc = OpToCondition(op);

  1255       __ cmpl(pri, alt);

  1256       __ j(cc, target);

  1257       break;

  1258     }

  1260     case OP_TRACKER_PUSH_C:

  1261     {

  1262       cell_t amount = readCell();

  1264       __ push(pri);

  1265       __ push(alt);

  1267       __ movl(eax, Operand(info, AMX_INFO_CONTEXT));

  1268       __ push(amount * 4);

  1269       __ push(eax);

  1270       __ call(ExternalAddress((void *)PushTracker));

  1271       __ addl(esp, 8);

  1272       __ testl(eax, eax);

  1273       __ j(not_zero, &extern_error_);

  1275       __ pop(alt);

  1276       __ pop(pri);

  1277       break;

  1278     }

  1280     case OP_TRACKER_POP_SETHEAP:

  1281     {

  1282       // Save registers.

  1283       __ push(pri);

  1284       __ push(alt);

  1286       // Get the context pointer and call the sanity checker.

  1287       __ push(info);

  1288       __ push(intptr_t(plugin_));

  1289       __ call(ExternalAddress((void *)PopTrackerAndSetHeap));

  1290       __ addl(esp, 8);

  1291       __ testl(eax, eax);

  1292       __ j(not_zero, &extern_error_);

  1294       __ pop(alt);

  1295       __ pop(pri);

  1296       break;

  1297     }

  1299     case OP_BREAK:

  1300     {

  1301       cell_t cip = uintptr_t(cip_ - 1) - uintptr_t(plugin_->pcode);

  1302       __ movl(Operand(info, AMX_INFO_CIP), cip);

  1303       break;

  1304     }

  1306     case OP_HALT:

  1307       __ align(16);

  1308       __ movl(tmp, Operand(info, AMX_INFO_RETVAL));

  1309       __ movl(Operand(ecx, 0), pri);

  1310       __ movl(pri, readCell());

  1311       __ jmp(&extern_error_);

  1312       break;

  1314     case OP_BOUNDS:

  1315     {

  1316       cell_t value = readCell();

  1317       __ cmpl(eax, value);

  1318       __ j(above, &error_bounds_);

  1319       break;

  1320     }

  1322     case OP_GENARRAY:

  1323     case OP_GENARRAY_Z:

  1324       emitGenArray(op == OP_GENARRAY_Z);

  1325       break;

  1327     case OP_CALL:

  1328       if (!emitCall())

  1329         return false;

  1330       break;

  1332     case OP_SYSREQ_C:

  1333     case OP_SYSREQ_N:

  1334       if (!emitNativeCall(op))

  1335         return false;

  1336       break;

  1338     case OP_SWITCH:

  1339       if (!emitSwitch())

  1340         return false;

  1341       break;

  1343     case OP_CASETBL:

  1344     {

  1345       size_t ncases = readCell();

  1347       // Two cells per case, and one extra cell for the default address.

  1348       cip_ += (ncases * 2) + 1;

  1349       break;

  1350     }

  1352     case OP_NOP:

  1353       break;

  1355     default:

  1356       error_ = SP_ERROR_INVALID_INSTRUCTION;

  1357       return false;

  1358   }

  1360   return true;

  1361 }

  1363 Label *

  1364 Compiler::labelAt(size_t offset)

  1365 {

  1366   if (offset % 4 != 0 ||

  1367       offset > plugin_->pcode_size ||

  1368       offset <= pcode_start_)

  1369   {

  1370     // If the jump target is misaligned, or out of pcode bounds, or is an

  1371     // address out of the function bounds, we abort. Unfortunately we can't

  1372     // test beyond the end of the function since we don't have a precursor

  1373     // pass (yet).

  1374     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1375     return NULL;

  1376   }

  1378   return &jump_map_[offset / sizeof(cell_t)];

  1379 }

  1381 void

  1382 Compiler::emitCheckAddress(Register reg)

  1383 {

  1384   // Check if we're in memory bounds.

  1385   __ cmpl(reg, plugin_->mem_size);

  1386   __ j(not_below, &error_memaccess_);

  1388   // Check if we're in the invalid region between hp and sp.

  1389   Label done;

  1390   __ cmpl(reg, Operand(info, AMX_INFO_HEAP));

  1391   __ j(below, &done);

  1392   __ lea(tmp, Operand(dat, reg, NoScale));

  1393   __ cmpl(tmp, stk);

  1394   __ j(below, &error_memaccess_);

  1395   __ bind(&done);

  1396 }

  1398 void

  1399 Compiler::emitGenArray(bool autozero)

  1400 {

  1401   cell_t val = readCell();

  1402   if (val == 1)

  1403   {

  1404     // flat array; we can generate this without indirection tables.

  1405     // Note that we can overwrite ALT because technically STACK should be destroying ALT

  1406     __ movl(alt, Operand(info, AMX_INFO_HEAP));

  1407     __ movl(tmp, Operand(stk, 0));

  1408     __ movl(Operand(stk, 0), alt);    // store base of the array into the stack.

  1409     __ lea(alt, Operand(alt, tmp, ScaleFour));

  1410     __ movl(Operand(info, AMX_INFO_HEAP), alt);

  1411     __ addl(alt, dat);

  1412     __ cmpl(alt, stk);

  1413     __ j(not_below, &error_heap_low_);

  1415     __ shll(tmp, 2);

  1416     __ push(tmp);

  1417     __ push(Operand(info, AMX_INFO_CONTEXT));

  1418     __ call(ExternalAddress((void *)PushTracker));

  1419     __ addl(esp, 4);

  1420     __ pop(tmp);

  1421     __ shrl(tmp, 2);

  1422     __ testl(eax, eax);

  1423     __ j(not_zero, &extern_error_);

  1425     if (autozero) {

  1426       // Note - tmp is ecx and still intact.

  1427       __ push(eax);

  1428       __ push(edi);

  1429       __ xorl(eax, eax);

  1430       __ movl(edi, Operand(stk, 0));

  1431       __ addl(edi, dat);

  1432       __ cld();

  1433       __ rep_stosd();

  1434       __ pop(edi);

  1435       __ pop(eax);

  1436     }

  1437   } else {

  1438     __ push(pri);

  1440     // int GenerateArray(sp_plugin_t, vars[], uint32_t, cell_t *, int, unsigned *);

  1441     __ push(autozero ? 1 : 0);

  1442     __ push(stk);

  1443     __ push(val);

  1444     __ push(info);

  1445     __ push(intptr_t(plugin_));

  1446     __ call(ExternalAddress((void *)GenerateArray));

  1447     __ addl(esp, 5 * sizeof(void *));

  1449     // restore pri to tmp

  1450     __ pop(tmp);

  1452     __ testl(eax, eax);

  1453     __ j(not_zero, &extern_error_);

  1455     // Move tmp back to pri, remove pushed args.

  1456     __ movl(pri, tmp);

  1457     __ addl(stk, (val - 1) * 4);

  1458   }

  1459 }

  1461 bool

  1462 Compiler::emitCall()

  1463 {

  1464   cell_t offset = readCell();

  1466   // If this offset looks crappy, i.e. not aligned or out of bounds, we just

  1467   // abort.

  1468   if (offset % 4 != 0 || uint32_t(offset) >= plugin_->pcode_size) {

  1469     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1470     return false;

  1471   }

  1473   // eax = context

  1474   // ecx = rp

  1475   __ movl(eax, Operand(info, AMX_INFO_CONTEXT));

  1476   __ movl(ecx, Operand(eax, offsetof(sp_context_t, rp)));

  1478   // Check if the return stack is used up.

  1479   __ cmpl(ecx, SP_MAX_RETURN_STACK);

  1480   __ j(not_below, &error_stack_low_);

  1482   // Add to the return stack.

  1483   uintptr_t cip = uintptr_t(cip_ - 2) - uintptr_t(plugin_->pcode);

  1484   __ movl(Operand(eax, ecx, ScaleFour, offsetof(sp_context_t, rstk_cips)), cip);

  1486   // Increment the return stack pointer.

  1487   __ addl(Operand(eax, offsetof(sp_context_t, rp)), 1);

  1489   // Store the CIP of the function we're about to call.

  1490   __ movl(Operand(info, AMX_INFO_CIP), offset);

  1492   JitFunction *fun = rt_->GetJittedFunctionByOffset(offset);

  1493   if (!fun) {

  1494     // Need to emit a delayed thunk.

  1495     CallThunk *thunk = new CallThunk(offset);

  1496     __ call(&thunk->call);

  1497     if (!thunks_.append(thunk))

  1498       return false;

  1499   } else {

  1500     // Function is already emitted, we can do a direct call.

  1501     __ call(ExternalAddress(fun->GetEntryAddress()));

  1502   }

  1504   // Restore the last cip.

  1505   __ movl(Operand(info, AMX_INFO_CIP), cip);

  1507   // Mark us as leaving the last frame.

  1508   __ movl(tmp, Operand(info, AMX_INFO_CONTEXT));

  1509   __ subl(Operand(tmp, offsetof(sp_context_t, rp)), 1);

  1510   return true;

  1511 }

  1513 void

  1514 Compiler::emitCallThunks()

  1515 {

  1516   for (size_t i = 0; i < thunks_.length(); i++) {

  1517     CallThunk *thunk = thunks_[i];

  1519     Label error;

  1520     __ bind(&thunk->call);

  1521     // Huge hack - get the return address, since that is the call that we

  1522     // need to patch.

  1523     __ movl(eax, Operand(esp, 0));

  1525     // Align the stack. Unfortunately this is a pretty careful dance based

  1526     // on the number of words we push (4 args, 1 saved esp), and someday

  1527     // we should look into automating this.

  1528     __ movl(edx, esp);

  1529     __ andl(esp, 0xfffffff0);

  1530     __ subl(esp, 12);

  1531     __ push(edx);

  1533     // Push arguments.

  1534     __ push(eax);

  1535     __ subl(esp, 4);

  1536     __ movl(Operand(esp, 0), esp);

  1537     __ push(thunk->pcode_offset);

  1538     __ push(intptr_t(rt_));

  1539     __ call(ExternalAddress((void *)CompileFromThunk));

  1540     __ movl(edx, Operand(esp, 8));

  1541     __ movl(esp, Operand(esp, 16));

  1542     __ testl(eax, eax);

  1543     __ j(not_zero, &error);

  1544     __ call(edx);

  1545     __ ret();

  1547     __ bind(&error);

  1548     __ movl(Operand(info, AMX_INFO_CIP), thunk->pcode_offset);

  1549     __ jmp(g_Jit.GetUniversalReturn());

  1550   }

  1551 }

  1553 cell_t

  1554 Compiler::readCell()

  1555 {

  1556   if (cip_ >= code_end_) {

  1557     error_= SP_ERROR_INVALID_INSTRUCTION;

  1558     return 0;

  1559   }

  1560   return *cip_++;

  1561 }

  1563 bool

  1564 Compiler::emitNativeCall(OPCODE op)

  1565 {

  1566   uint32_t native_index = readCell();

  1568   if (native_index >= plugin_->num_natives) {

  1569     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1570     return false;

  1571   }

  1573   uint32_t num_params;

  1574   if (op == OP_SYSREQ_N) {

  1575     num_params = readCell();

  1577     // See if we can get a replacement opcode. If we can, then recursively

  1578     // call emitOp() to generate it. Note: it's important that we do this

  1579     // before generating any code for the SYSREQ.N.

  1580     unsigned replacement = rt_->GetNativeReplacement(native_index);

  1581     if (replacement != OP_NOP)

  1582       return emitOp((OPCODE)replacement);

  1584     // Store the number of parameters.

  1585     __ movl(Operand(stk, -4), num_params);

  1586     __ subl(stk, 4);

  1587   }

  1589   // Save registers.

  1590   __ push(edx);

  1592   // Push the parameters for the C++ function.

  1593   __ push(stk);

  1594   __ push(native_index);

  1596   // Relocate all our absolute junk to be dat-relative, and store it all back

  1597   // into the context.

  1598   __ movl(eax, Operand(info, AMX_INFO_CONTEXT));

  1599   __ movl(ecx, Operand(info, AMX_INFO_HEAP));

  1600   __ movl(Operand(eax, offsetof(sp_context_t, hp)), ecx);

  1601   __ subl(stk, dat);

  1602   __ movl(Operand(eax, offsetof(sp_context_t, sp)), stk);

  1603   __ movl(ecx, Operand(info, AMX_INFO_FRAME));

  1604   __ movl(Operand(eax, offsetof(sp_context_t, frm)), ecx);

  1606   // Push context and make the call.

  1607   __ push(eax);

  1608   __ call(ExternalAddress((void *)NativeCallback));

  1610   // Check for errors.

  1611   __ movl(ecx, Operand(info, AMX_INFO_CONTEXT));

  1612   __ movl(ecx, Operand(ecx, offsetof(sp_context_t, n_err)));

  1613   __ testl(ecx, ecx);

  1614   __ j(not_zero, &extern_error_);

  1616   // Restore local state.

  1617   __ addl(stk, dat);

  1618   __ addl(esp, 12);

  1619   __ pop(edx);

  1621   if (op == OP_SYSREQ_N) {

  1622     // Pop the stack. Do not check the margins.

  1623     __ addl(stk, (num_params + 1) * sizeof(cell_t));

  1624   }

  1625   return true;

  1626 }

  1628 bool

  1629 Compiler::emitSwitch()

  1630 {

  1631   cell_t offset = readCell();

  1632   if (!labelAt(offset))

  1633     return false;

  1635   cell_t *tbl = (cell_t *)((char *)plugin_->pcode + offset + sizeof(cell_t));

  1637   struct Entry {

  1638     cell_t val;

  1639     cell_t offset;

  1640   };

  1642   size_t ncases = *tbl++;

  1644   Label *defaultCase = labelAt(*tbl);

  1645   if (!defaultCase)

  1646     return false;

  1648   // Degenerate - 0 cases.

  1649   if (!ncases) {

  1650     __ jmp(defaultCase);

  1651     return true;

  1652   }

  1654   Entry *cases = (Entry *)(tbl + 1);

  1656   // Degenerate - 1 case.

  1657   if (ncases == 1) {

  1658     Label *maybe = labelAt(cases[0].offset);

  1659     if (!maybe)

  1660       return false;

  1661     __ cmpl(pri, cases[0].val);

  1662     __ j(equal, maybe);

  1663     __ jmp(defaultCase);

  1664     return true;

  1665   }

  1667   // We have two or more cases, so let's generate a full switch. Decide

  1668   // whether we'll make an if chain, or a jump table, based on whether

  1669   // the numbers are strictly sequential.

  1670   bool sequential = true;

  1671   {

  1672     cell_t first = cases[0].val;

  1673     cell_t last = first;

  1674     for (size_t i = 1; i < ncases; i++) {

  1675       if (cases[i].val != ++last) {

  1676         sequential = false;

  1677         break;

  1678       }

  1679     }

  1680   }

  1682   // First check whether the bounds are correct: if (a < LOW || a > HIGH);

  1683   // this check is valid whether or not we emit a sequential-optimized switch.

  1684   cell_t low = cases[0].val;

  1685   if (low != 0) {

  1686     // negate it so we'll get a lower bound of 0.

  1687     low = -low;

  1688     __ lea(tmp, Operand(pri, low));

  1689   } else {

  1690     __ movl(tmp, pri);

  1691   }

  1693   cell_t high = abs(cases[0].val - cases[ncases - 1].val);

  1694   __ cmpl(tmp, high);

  1695   __ j(above, defaultCase);

  1697   if (sequential) {

  1698     // Optimized table version. The tomfoolery below is because we only have

  1699     // one free register... it seems unlikely pri or alt will be used given

  1700     // that we're at the end of a control-flow point, but we'll play it safe.

  1701     DataLabel table;

  1702     __ push(eax);

  1703     __ movl(eax, &table);

  1704     __ movl(ecx, Operand(eax, ecx, ScaleFour));

  1705     __ pop(eax);

  1706     __ jmp(ecx);

  1708     __ bind(&table);

  1709     for (size_t i = 0; i < ncases; i++) {

  1710       Label *label = labelAt(cases[i].offset);

  1711       if (!label)

  1712         return false;

  1713       __ emit_absolute_address(label);

  1714     }

  1715   } else {

  1716     // Slower version. Go through each case and generate a check.

  1717     for (size_t i = 0; i < ncases; i++) {

  1718       Label *label = labelAt(cases[i].offset);

  1719       if (!label)

  1720         return false;

  1721       __ cmpl(pri, cases[i].val);

  1722       __ j(equal, label);

  1723     }

  1724     __ jmp(defaultCase);

  1725   }

  1726   return true;

  1727 }

  1729 void

  1730 Compiler::emitErrorPath(Label *dest, int code)

  1731 {

  1732   if (dest->used()) {

  1733     __ bind(dest);

  1734     __ movl(eax, code);

  1735     __ jmp(g_Jit.GetUniversalReturn());

  1736   }

  1737 }

  1739 void

  1740 Compiler::emitErrorPaths()

  1741 {

  1742   emitErrorPath(&error_divide_by_zero_, SP_ERROR_DIVIDE_BY_ZERO);

  1743   emitErrorPath(&error_stack_low_, SP_ERROR_STACKLOW);

  1744   emitErrorPath(&error_stack_min_, SP_ERROR_STACKMIN);

  1745   emitErrorPath(&error_bounds_, SP_ERROR_ARRAY_BOUNDS);

  1746   emitErrorPath(&error_memaccess_, SP_ERROR_MEMACCESS);

  1747   emitErrorPath(&error_heap_low_, SP_ERROR_HEAPLOW);

  1748   emitErrorPath(&error_heap_min_, SP_ERROR_HEAPMIN);

  1749   emitErrorPath(&error_integer_overflow_, SP_ERROR_INTEGER_OVERFLOW);

  1751   if (extern_error_.used()) {

  1752     __ bind(&extern_error_);

  1753     __ movl(eax, Operand(info, AMX_INFO_CONTEXT));

  1754     __ movl(eax, Operand(eax, offsetof(sp_context_t, n_err)));

  1755     __ jmp(g_Jit.GetUniversalReturn());

  1756   }

  1757 }

  1759 static void *

  1760 GenerateEntry(void **retp)

  1761 {

  1762   AssemblerX86 masm;

  1764   // Variables we're passed in:

  1765   //  InfoVars *vars, void *entry, uint8_t *memory

  1767   __ push(ebp);

  1768   __ movl(ebp, esp);

  1770   __ push(esi);

  1771   __ push(edi);

  1772   __ push(ebx);

  1774   __ movl(esi, Operand(ebp, 8 + 4 * 0));

  1775   __ movl(ecx, Operand(ebp, 8 + 4 * 1));

  1776   __ movl(eax, Operand(ebp, 8 + 4 * 2));

  1777   __ movl(edx, Operand(esi, AMX_INFO_CONTEXT));

  1779   // Set up run-time registers.

  1780   __ movl(edi, Operand(edx, offsetof(sp_context_t, sp)));

  1781   __ addl(edi, eax);

  1782   __ movl(ebp, eax);

  1783   __ movl(ebx, edi);

  1784   __ movl(Operand(esi, AMX_INFO_NSTACK), esp);

  1786   // Align the stack.

  1787   __ andl(esp, 0xfffffff0);

  1789   // Call into plugin (align the stack first).

  1790   __ call(ecx);

  1792   __ movl(ecx, Operand(info, AMX_INFO_RETVAL));

  1793   __ movl(Operand(ecx, 0), pri);

  1794   __ movl(eax, SP_ERROR_NONE);

  1796   // If stuff goes wrong, it'll jump directly to here.

  1797   Label error;

  1798   __ bind(&error);

  1799   __ movl(esp, Operand(info, AMX_INFO_NSTACK));

  1800   __ movl(ecx, Operand(info, AMX_INFO_CONTEXT));

  1801   __ subl(stk, dat);

  1802   __ movl(Operand(ecx, offsetof(sp_context_t, sp)), stk);

  1804   __ pop(ebx);

  1805   __ pop(edi);

  1806   __ pop(esi);

  1807   __ pop(ebp);

  1808   __ ret();

  1810   void *code = LinkCode(masm);

  1811   if (!code)

  1812     return NULL;

  1814   *retp = reinterpret_cast<uint8_t *>(code) + error.offset();

  1815   return code;

  1816 }

  1818 ICompilation *JITX86::ApplyOptions(ICompilation *_IN, ICompilation *_OUT)

  1819 {

  1820   if (_IN == NULL)

  1821     return _OUT;

  1823   CompData *_in = (CompData * )_IN;

  1824   CompData *_out = (CompData * )_OUT;

  1826   _in->inline_level = _out->inline_level;

  1827   _in->profile = _out->profile;

  1829   _out->Abort();

  1830   return _in;

  1831 }

  1833 JITX86::JITX86()

  1834 {

  1835   m_pJitEntry = NULL;

  1836 }

  1838 bool JITX86::InitializeJIT()

  1839 {

  1840   g_pCodeCache = KE_CreateCodeCache();

  1842   m_pJitEntry = GenerateEntry(&m_pJitReturn);

  1843   if (!m_pJitEntry)

  1844     return false;

  1846   MacroAssemblerX86 masm;

  1847   MacroAssemblerX86::GenerateFeatureDetection(masm);

  1848   void *code = LinkCode(masm);

  1849   if (!code)

  1850     return false;

  1851   MacroAssemblerX86::RunFeatureDetection(code);

  1852   KE_FreeCode(g_pCodeCache, code);

  1854   return true;

  1855 }

  1857 void

  1858 JITX86::ShutdownJIT()

  1859 {

  1860   KE_DestroyCodeCache(g_pCodeCache);

  1861 }

  1863 JitFunction *

  1864 JITX86::CompileFunction(BaseRuntime *prt, cell_t pcode_offs, int *err)

  1865 {

  1866   Compiler cc(prt, pcode_offs);

  1867   JitFunction *fun = cc.emit(err);

  1868   if (!fun)

  1869     return NULL;

  1870   prt->AddJittedFunction(fun);

  1871   return fun;

  1872 }

  1874 void

  1875 JITX86::SetupContextVars(BaseRuntime *runtime, BaseContext *pCtx, sp_context_t *ctx)

  1876 {

  1877   tracker_t *trk = new tracker_t;

  1879   ctx->vm[JITVARS_TRACKER] = trk;

  1880   ctx->vm[JITVARS_BASECTX] = pCtx; /* GetDefaultContext() is not constructed yet */

  1881   ctx->vm[JITVARS_PROFILER] = g_engine2.GetProfiler();

  1882   ctx->vm[JITVARS_PLUGIN] = const_cast<sp_plugin_t *>(runtime->plugin());

  1884   trk->pBase = (ucell_t *)malloc(1024);

  1885   trk->pCur = trk->pBase;

  1886   trk->size = 1024 / sizeof(cell_t);

  1887 }

  1889 SPVM_NATIVE_FUNC

  1890 JITX86::CreateFakeNative(SPVM_FAKENATIVE_FUNC callback, void *pData)

  1891 {

  1892   AssemblerX86 masm;

  1894   __ push(ebx);

  1895   __ push(edi);

  1896   __ push(esi);

  1897   __ movl(edi, Operand(esp, 16)); // store ctx

  1898   __ movl(esi, Operand(esp, 20)); // store params

  1899   __ movl(ebx, esp);

  1900   __ andl(esp, 0xfffffff0);

  1901   __ subl(esp, 4);

  1903   __ push(intptr_t(pData));

  1904   __ push(esi);

  1905   __ push(edi);

  1906   __ call(ExternalAddress((void *)callback));

  1907   __ movl(esp, ebx);

  1908   __ pop(esi);

  1909   __ pop(edi);

  1910   __ pop(ebx);

  1911   __ ret();

  1913   return (SPVM_NATIVE_FUNC)LinkCode(masm);

  1914 }

  1916 void JITX86::DestroyFakeNative(SPVM_NATIVE_FUNC func)

  1917 {

  1918   KE_FreeCode(g_pCodeCache, (void *)func);

  1919 }

  1921 ICompilation *JITX86::StartCompilation()

  1922 {

  1923   return new CompData;

  1924 }

  1926 ICompilation *JITX86::StartCompilation(BaseRuntime *runtime)

  1927 {

  1928   return new CompData;

  1929 }

  1931 void CompData::Abort()

  1932 {

  1933   delete this;

  1934 }

  1936 void JITX86::FreeContextVars(sp_context_t *ctx)

  1937 {

  1938   free(((tracker_t *)(ctx->vm[JITVARS_TRACKER]))->pBase);

  1939   delete (tracker_t *)ctx->vm[JITVARS_TRACKER];

  1940 }

  1942 bool CompData::SetOption(const char *key, const char *val)

  1943 {

  1944   if (strcmp(key, SP_JITCONF_DEBUG) == 0)

  1945     return true;

  1946   if (strcmp(key, SP_JITCONF_PROFILE) == 0) {

  1947     profile = atoi(val);

  1949     /** Callbacks must be profiled to profile functions! */

  1950     if ((profile & SP_PROF_FUNCTIONS) == SP_PROF_FUNCTIONS)

  1951       profile |= SP_PROF_CALLBACKS;

  1953     return true;

  1954   }

  1956   return false;

  1957 }

  1959 typedef int (*JIT_EXECUTE)(InfoVars *vars, void *addr, uint8_t *memory);

  1960 int JITX86::InvokeFunction(BaseRuntime *runtime, JitFunction *fn, cell_t *result)

  1961 {

  1962   int err;

  1963   JIT_EXECUTE pfn;

  1964   sp_context_t *ctx;

  1965   InfoVars vars;

  1967   ctx = runtime->GetBaseContext()->GetCtx();

  1969   vars.frm = ctx->sp;

  1970   vars.hp = ctx->hp;

  1971   vars.rval = result;

  1972   vars.ctx = ctx;

  1973   vars.cip = fn->GetPCodeAddress();

  1974   /* vars.esp will be set in the entry code */

  1976   pfn = (JIT_EXECUTE)m_pJitEntry;

  1977   err = pfn(&vars, fn->GetEntryAddress(), runtime->plugin()->memory);

  1979   ctx->hp = vars.hp;

  1980   ctx->err_cip = vars.cip;

  1982   return err;

  1983 }

  1985 void *JITX86::AllocCode(size_t size)

  1986 {

  1987   return Knight::KE_AllocCode(g_pCodeCache, size);

  1988 }

  1990 void JITX86::FreeCode(void *code)

  1991 {

  1992   KE_FreeCode(g_pCodeCache, code);

  1993 }

author	David Anderson <dvander@alliedmods.net>
	Sun Aug 11 11:33:29 2013 -0700 (2013-08-11)
changeset 3987	05ab44d7ec01
parent 3986	f49be7182920
child 3988	f947f2955ae0
permissions	-rw-r--r--