sourcemod-central: sourcepawn/jit/x86/jit

     1 /**

     2  * vim: set ts=2 sw=2 tw=99 et:

     3  * =============================================================================

     4  * SourceMod

     5  * Copyright (C) 2004-2008 AlliedModders LLC.  All rights reserved.

     6  * =============================================================================

     7  *

     8  * This program is free software; you can redistribute it and/or modify it under

     9  * the terms of the GNU General Public License, version 3.0, as published by the

    10  * Free Software Foundation.

    11  *

    12  * This program is distributed in the hope that it will be useful, but WITHOUT

    13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    14  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more

    15  * details.

    16  *

    17  * You should have received a copy of the GNU General Public License along with

    18  * this program.  If not, see <http://www.gnu.org/licenses/>.

    19  *

    20  * As a special exception, AlliedModders LLC gives you permission to link the

    21  * code of this program (as well as its derivative works) to "Half-Life 2," the

    22  * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software

    23  * by the Valve Corporation.  You must obey the GNU General Public License in

    24  * all respects for all other code used.  Additionally, AlliedModders LLC grants

    25  * this exception to all derivative works.  AlliedModders LLC defines further

    26  * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),

    27  * or <http://www.sourcemod.net/license.php>.

    28  *

    29  * Version: $Id$

    30  */

    32 #include <string.h>

    33 #include <stdlib.h>

    34 #include <assert.h>

    35 #include "jit_x86.h"

    36 #include "../sp_vm_engine.h"

    37 #include "../engine2.h"

    38 #include "../BaseRuntime.h"

    39 #include "../sp_vm_basecontext.h"

    41 using namespace Knight;

    43 #if defined USE_UNGEN_OPCODES

    44 #include "ungen_opcodes.h"

    45 #endif

    47 #define __ masm.

    49 JITX86 g_Jit;

    50 KeCodeCache *g_pCodeCache = NULL;

    51 ISourcePawnEngine *engine = &g_engine1;

    53 static inline void *

    54 LinkCode(AssemblerX86 &masm)

    55 {

    56   if (masm.outOfMemory())

    57     return NULL;

    59   void *code = Knight::KE_AllocCode(g_pCodeCache, masm.length());

    60   if (!code)

    61     return NULL;

    63   masm.emitToExecutableMemory(code);

    64   return code;

    65 }

    67 static inline ConditionCode

    68 OpToCondition(OPCODE op)

    69 {

    70   switch (op) {

    71    case OP_EQ:

    72    case OP_JEQ:

    73     return equal;

    74    case OP_NEQ:

    75    case OP_JNEQ:

    76     return not_equal;

    77    case OP_SLESS:

    78    case OP_JSLESS:

    79     return less;

    80    case OP_SLEQ:

    81    case OP_JSLEQ:

    82     return less_equal;

    83    case OP_SGRTR:

    84    case OP_JSGRTR:

    85     return greater;

    86    case OP_SGEQ:

    87    case OP_JSGEQ:

    88     return greater_equal;

    89    default:

    90     assert(false);

    91     return negative;

    92   }

    93 }

    95 struct array_creation_t

    96 {

    97   const cell_t *dim_list;     /* Dimension sizes */

    98   cell_t dim_count;           /* Number of dimensions */

    99   cell_t *data_offs;          /* Current offset AFTER the indirection vectors (data) */

   100   cell_t *base;               /* array base */

   101 };

   103 static cell_t

   104 GenerateInnerArrayIndirectionVectors(array_creation_t *ar, int dim, cell_t cur_offs)

   105 {

   106   cell_t write_offs = cur_offs;

   107   cell_t *data_offs = ar->data_offs;

   109   cur_offs += ar->dim_list[dim];

   111   // Dimension n-x where x > 2 will have sub-vectors.

   112   // Otherwise, we just need to reference the data section.

   113   if (ar->dim_count > 2 && dim < ar->dim_count - 2) {

   114     // For each index at this dimension, write offstes to our sub-vectors.

   115     // After we write one sub-vector, we generate its sub-vectors recursively.

   116     // At the end, we're given the next offset we can use.

   117     for (int i = 0; i < ar->dim_list[dim]; i++) {

   118       ar->base[write_offs] = (cur_offs - write_offs) * sizeof(cell_t);

   119       write_offs++;

   120       cur_offs = GenerateInnerArrayIndirectionVectors(ar, dim + 1, cur_offs);

   121     }

   122   } else {

   123     // In this section, there are no sub-vectors, we need to write offsets

   124     // to the data.  This is separate so the data stays in one big chunk.

   125     // The data offset will increment by the size of the last dimension,

   126     // because that is where the data is finally computed as.

   127     for (int i = 0; i < ar->dim_list[dim]; i++) {

   128       ar->base[write_offs] = (*data_offs - write_offs) * sizeof(cell_t);

   129       write_offs++;

   130       *data_offs = *data_offs + ar->dim_list[dim + 1];

   131     }

   132   }

   134   return cur_offs;

   135 }

   137 static cell_t

   138 calc_indirection(const array_creation_t *ar, cell_t dim)

   139 {

   140   cell_t size = ar->dim_list[dim];

   141   if (dim < ar->dim_count - 2)

   142     size += ar->dim_list[dim] * calc_indirection(ar, dim + 1);

   143   return size;

   144 }

   146 static cell_t

   147 GenerateArrayIndirectionVectors(cell_t *arraybase, cell_t dims[], cell_t _dimcount, bool autozero)

   148 {

   149   array_creation_t ar;

   150   cell_t data_offs;

   152   /* Reverse the dimensions */

   153   cell_t dim_list[sDIMEN_MAX];

   154   int cur_dim = 0;

   155   for (int i = _dimcount - 1; i >= 0; i--)

   156     dim_list[cur_dim++] = dims[i];

   158   ar.base = arraybase;

   159   ar.dim_list = dim_list;

   160   ar.dim_count = _dimcount;

   161   ar.data_offs = &data_offs;

   163   data_offs = calc_indirection(&ar, 0);

   164   GenerateInnerArrayIndirectionVectors(&ar, 0, 0);

   165   return data_offs;

   166 }

   168 static int

   169 PopTrackerAndSetHeap(BaseRuntime *rt)

   170 {

   171   sp_context_t *ctx = rt->GetBaseContext()->GetCtx();

   172   tracker_t *trk = (tracker_t *)(ctx->vm[JITVARS_TRACKER]);

   173   assert(trk->pCur > trk->pBase);

   175   trk->pCur--;

   176   if (trk->pCur < trk->pBase)

   177     return SP_ERROR_TRACKER_BOUNDS;

   179   ucell_t amt = *trk->pCur;

   180   if (amt > (ctx->hp - rt->plugin()->data_size))

   181     return SP_ERROR_HEAPMIN;

   183   ctx->hp -= amt;

   184   return SP_ERROR_NONE;

   185 }

   187 static int

   188 PushTracker(sp_context_t *ctx, size_t amount)

   189 {

   190   tracker_t *trk = (tracker_t *)(ctx->vm[JITVARS_TRACKER]);

   192   if ((size_t)(trk->pCur - trk->pBase) >= trk->size)

   193     return SP_ERROR_TRACKER_BOUNDS;

   195   if (trk->pCur + 1 - (trk->pBase + trk->size) == 0) {

   196     size_t disp = trk->size - 1;

   197     trk->size *= 2;

   198     trk->pBase = (ucell_t *)realloc(trk->pBase, trk->size * sizeof(cell_t));

   200     if (!trk->pBase)

   201       return SP_ERROR_TRACKER_BOUNDS;

   203     trk->pCur = trk->pBase + disp;

   204   }

   206   *trk->pCur++ = amount;

   207   return SP_ERROR_NONE;

   208 }

   210 static int

   211 GenerateArray(BaseRuntime *rt, uint32_t argc, cell_t *argv, int autozero)

   212 {

   213   sp_context_t *ctx = rt->GetBaseContext()->GetCtx();

   215   // Calculate how many cells are needed.

   216   if (argv[0] <= 0)

   217     return SP_ERROR_ARRAY_TOO_BIG;

   219   uint32_t dim = 1;         // second to last dimension

   220   uint32_t cells = argv[0];

   222   for (uint32_t dim = 1; dim < argc; dim++) {

   223     cell_t dimsize = argv[dim];

   224     if (dimsize <= 0)

   225       return SP_ERROR_ARRAY_TOO_BIG;

   226     if (!ke::IsUint32MultiplySafe(cells, dimsize))

   227       return SP_ERROR_ARRAY_TOO_BIG;

   228     cells *= uint32_t(dimsize);

   229     if (!ke::IsUint32AddSafe(cells, dimsize))

   230       return SP_ERROR_ARRAY_TOO_BIG;

   231     cells += uint32_t(dimsize);

   232   }

   234   if (!ke::IsUint32MultiplySafe(cells, 4))

   235     return SP_ERROR_ARRAY_TOO_BIG;

   237   uint32_t bytes = cells * 4;

   238   if (!ke::IsUint32AddSafe(ctx->hp, bytes))

   239     return SP_ERROR_ARRAY_TOO_BIG;

   241   uint32_t new_hp = ctx->hp + bytes;

   242   cell_t *dat_hp = reinterpret_cast<cell_t *>(rt->plugin()->memory + new_hp);

   244   // argv, coincidentally, is STK.

   245   if (dat_hp >= argv - STACK_MARGIN)

   246     return SP_ERROR_HEAPLOW;

   248   if (int err = PushTracker(rt->GetBaseContext()->GetCtx(), bytes))

   249     return err;

   251   cell_t *base = reinterpret_cast<cell_t *>(rt->plugin()->memory + ctx->hp);

   252   cell_t offs = GenerateArrayIndirectionVectors(base, argv, argc, autozero);

   253   assert(size_t(offs) == cells);

   255   argv[argc - 1] = ctx->hp;

   256   ctx->hp = new_hp;

   257   return SP_ERROR_NONE;

   258 }

   260 #if !defined NDEBUG

   261 static const char *

   262 GetFunctionName(const sp_plugin_t *plugin, uint32_t offs)

   263 {

   264   if (!plugin->debug.unpacked) {

   265     uint32_t max, iter;

   266     sp_fdbg_symbol_t *sym;

   267     sp_fdbg_arraydim_t *arr;

   268     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   270     max = plugin->debug.syms_num;

   271     for (iter = 0; iter < max; iter++) {

   272       sym = (sp_fdbg_symbol_t *)cursor;

   274       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   275         return plugin->debug.stringbase + sym->name;

   277       if (sym->dimcount > 0) {

   278         cursor += sizeof(sp_fdbg_symbol_t);

   279         arr = (sp_fdbg_arraydim_t *)cursor;

   280         cursor += sizeof(sp_fdbg_arraydim_t) * sym->dimcount;

   281         continue;

   282       }

   284       cursor += sizeof(sp_fdbg_symbol_t);

   285     }

   286   } else {

   287     uint32_t max, iter;

   288     sp_u_fdbg_symbol_t *sym;

   289     sp_u_fdbg_arraydim_t *arr;

   290     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   292     max = plugin->debug.syms_num;

   293     for (iter = 0; iter < max; iter++) {

   294       sym = (sp_u_fdbg_symbol_t *)cursor;

   296       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   297         return plugin->debug.stringbase + sym->name;

   299       if (sym->dimcount > 0) {

   300         cursor += sizeof(sp_u_fdbg_symbol_t);

   301         arr = (sp_u_fdbg_arraydim_t *)cursor;

   302         cursor += sizeof(sp_u_fdbg_arraydim_t) * sym->dimcount;

   303         continue;

   304       }

   306       cursor += sizeof(sp_u_fdbg_symbol_t);

   307     }

   308   }

   310   return NULL;

   311 }

   312 #endif

   314 static int

   315 CompileFromThunk(BaseRuntime *runtime, cell_t pcode_offs, void **addrp, char *pc)

   316 {

   317   JitFunction *fn = runtime->GetJittedFunctionByOffset(pcode_offs);

   318   if (!fn) {

   319     int err;

   320     fn = g_Jit.CompileFunction(runtime, pcode_offs, &err);

   321     if (!fn)

   322       return err;

   323   }

   325 #if !defined NDEBUG

   326   g_engine1.GetDebugHook()->OnDebugSpew(

   327       "Patching thunk to %s::%s\n",

   328       runtime->plugin()->name,

   329       GetFunctionName(runtime->plugin(), pcode_offs));

   330 #endif

   332   *addrp = fn->GetEntryAddress();

   334   /* Right now, we always keep the code RWE */

   335   *(intptr_t *)(pc - 4) = intptr_t(fn->GetEntryAddress()) - intptr_t(pc);

   336   return SP_ERROR_NONE;

   337 }

   339 static cell_t

   340 NativeCallback(sp_context_t *ctx, ucell_t native_idx, cell_t *params)

   341 {

   342   sp_native_t *native;

   343   cell_t save_sp = ctx->sp;

   344   cell_t save_hp = ctx->hp;

   345   sp_plugin_t *pl = (sp_plugin_t *)(ctx->vm[JITVARS_PLUGIN]);

   347   ctx->n_idx = native_idx;

   349   if (ctx->hp < (cell_t)pl->data_size) {

   350     ctx->n_err = SP_ERROR_HEAPMIN;

   351     return 0;

   352   }

   354   if (ctx->hp + STACK_MARGIN > ctx->sp) {

   355     ctx->n_err = SP_ERROR_STACKLOW;

   356     return 0;

   357   }

   359   if ((uint32_t)ctx->sp >= pl->mem_size) {

   360     ctx->n_err = SP_ERROR_STACKMIN;

   361     return 0;

   362   }

   364   native = &pl->natives[native_idx];

   366   if (native->status == SP_NATIVE_UNBOUND) {

   367     ctx->n_err = SP_ERROR_INVALID_NATIVE;

   368     return 0;

   369   }

   371   cell_t result = native->pfn(GET_CONTEXT(ctx), params);

   373   if (ctx->n_err != SP_ERROR_NONE)

   374     return result;

   376   if (save_sp != ctx->sp) {

   377     ctx->n_err = SP_ERROR_STACKLEAK;

   378     return result;

   379   }

   380   if (save_hp != ctx->hp) {

   381     ctx->n_err = SP_ERROR_HEAPLEAK;

   382     return result;

   383   }

   385   return result;

   386 }

   388 Compiler::Compiler(BaseRuntime *rt, cell_t pcode_offs)

   389   : rt_(rt),

   390     plugin_(rt->plugin()),

   391     error_(SP_ERROR_NONE),

   392     pcode_start_(pcode_offs),

   393     code_start_(reinterpret_cast<cell_t *>(plugin_->pcode + pcode_start_)),

   394     cip_(code_start_),

   395     code_end_(reinterpret_cast<cell_t *>(plugin_->pcode + plugin_->pcode_size))

   396 {

   397   size_t nmaxops = plugin_->pcode_size / sizeof(cell_t) + 1;

   398   jump_map_ = new Label[nmaxops];

   399 }

   401 Compiler::~Compiler()

   402 {

   403   delete [] jump_map_;

   404 }

   406 JitFunction *

   407 Compiler::emit(int *errp)

   408 {

   409   if (cip_ >= code_end_ || *cip_ != OP_PROC) {

   410     *errp = SP_ERROR_INVALID_INSTRUCTION;

   411     return NULL;

   412   }

   414 #if !defined NDEBUG

   415   g_engine1.GetDebugHook()->OnDebugSpew(

   416       "Compiling function %s::%s\n",

   417       plugin_->name,

   418       GetFunctionName(plugin_, pcode_start_));

   420   SpewOpcode(plugin_, code_start_, cip_);

   421 #endif

   423   cell_t *codeseg = reinterpret_cast<cell_t *>(plugin_->pcode);

   425   cip_++;

   426   if (!emitOp(OP_PROC)) {

   427       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   428       return NULL;

   429   }

   431   while (cip_ < code_end_) {

   432     // If we reach the end of this function, or the beginning of a new

   433     // procedure, then stop.

   434     if (*cip_ == OP_PROC || *cip_ == OP_ENDPROC)

   435       break;

   437 #if !defined NDEBUG

   438     SpewOpcode(plugin_, code_start_, cip_);

   439 #endif

   441     // We assume every instruction is a jump target, so before emitting

   442     // an opcode, we bind its corresponding label.

   443     __ bind(&jump_map_[cip_ - codeseg]);

   445     OPCODE op = (OPCODE)readCell();

   446     if (!emitOp(op) || error_ != SP_ERROR_NONE) {

   447       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   448       return NULL;

   449     }

   450   }

   452   emitCallThunks();

   453   emitErrorPaths();

   455   void *code = LinkCode(masm);

   456   if (!code) {

   457     *errp = SP_ERROR_OUT_OF_MEMORY;

   458     return NULL;

   459   }

   461   return new JitFunction(code, pcode_start_);

   462 }

   464 bool

   465 Compiler::emitOp(OPCODE op)

   466 {

   467   switch (op) {

   468     case OP_MOVE_PRI:

   469       __ movl(pri, alt);

   470       break;

   472     case OP_MOVE_ALT:

   473       __ movl(alt, pri);

   474       break;

   476     case OP_XCHG:

   477       __ xchgl(pri, alt);

   478       break;

   480     case OP_ZERO:

   481     {

   482       cell_t offset = readCell();

   483       __ movl(Operand(dat, offset), 0);

   484       break;

   485     }

   487     case OP_ZERO_S:

   488     {

   489       cell_t offset = readCell();

   490       __ movl(Operand(frm, offset), 0);

   491       break;

   492     }

   494     case OP_PUSH_PRI:

   495     case OP_PUSH_ALT:

   496     {

   497       Register reg = (op == OP_PUSH_PRI) ? pri : alt;

   498       __ movl(Operand(stk, -4), reg);

   499       __ subl(stk, 4);

   500       break;

   501     }

   503     case OP_PUSH_C:

   504     case OP_PUSH2_C:

   505     case OP_PUSH3_C:

   506     case OP_PUSH4_C:

   507     case OP_PUSH5_C:

   508     {

   509       int n = 1;

   510       if (op >= OP_PUSH2_C)

   511         n = ((op - OP_PUSH2_C) / 4) + 2;

   513       int i = 1;

   514       do {

   515         cell_t val = readCell();

   516         __ movl(Operand(stk, -(4 * i)), val);

   517       } while (i++ < n);

   518       __ subl(stk, 4 * n);

   519       break;

   520     }

   522     case OP_PUSH_ADR:

   523     case OP_PUSH2_ADR:

   524     case OP_PUSH3_ADR:

   525     case OP_PUSH4_ADR:

   526     case OP_PUSH5_ADR:

   527     {

   528       int n = 1;

   529       if (op >= OP_PUSH2_ADR)

   530         n = ((op - OP_PUSH2_ADR) / 4) + 2;

   532       int i = 1;

   534       // We temporarily relocate FRM to be a local address instead of an

   535       // absolute address.

   536       __ subl(frm, dat);

   537       do {

   538         cell_t offset = readCell();

   539         __ lea(tmp, Operand(frm, offset));

   540         __ movl(Operand(stk, -(4 * i)), tmp);

   541       } while (i++ < n);

   542       __ subl(stk, 4 * n);

   543       __ addl(frm, dat);

   544       break;

   545     }

   547     case OP_PUSH_S:

   548     case OP_PUSH2_S:

   549     case OP_PUSH3_S:

   550     case OP_PUSH4_S:

   551     case OP_PUSH5_S:

   552     {

   553       int n = 1;

   554       if (op >= OP_PUSH2_S)

   555         n = ((op - OP_PUSH2_S) / 4) + 2;

   557       int i = 1;

   558       do {

   559         cell_t offset = readCell();

   560         __ movl(tmp, Operand(frm, offset));

   561         __ movl(Operand(stk, -(4 * i)), tmp);

   562       } while (i++ < n);

   563       __ subl(stk, 4 * n);

   564       break;

   565     }

   567     case OP_PUSH:

   568     case OP_PUSH2:

   569     case OP_PUSH3:

   570     case OP_PUSH4:

   571     case OP_PUSH5:

   572     {

   573       int n = 1;

   574       if (op >= OP_PUSH2)

   575         n = ((op - OP_PUSH2) / 4) + 2;

   577       int i = 1;

   578       do {

   579         cell_t offset = readCell();

   580         __ movl(tmp, Operand(dat, offset));

   581         __ movl(Operand(stk, -(4 * i)), tmp);

   582       } while (i++ < n);

   583       __ subl(stk, 4 * n);

   584       break;

   585     }

   587     case OP_ZERO_PRI:

   588       __ xorl(pri, pri);

   589       break;

   591     case OP_ZERO_ALT:

   592       __ xorl(alt, alt);

   593       break;

   595     case OP_ADD:

   596       __ addl(pri, alt);

   597       break;

   599     case OP_SUB:

   600       __ subl(pri, alt);

   601       break;

   603     case OP_SUB_ALT:

   604       __ movl(tmp, alt);

   605       __ subl(tmp, pri);

   606       __ movl(pri, tmp);

   607       break;

   609     case OP_PROC:

   610       // Push the old frame onto the stack.

   611       __ movl(tmp, Operand(frmAddr()));

   612       __ movl(Operand(stk, -4), tmp);

   613       __ subl(stk, 8);    // extra unused slot for non-existant CIP

   615       // Get and store the new frame.

   616       __ movl(tmp, stk);

   617       __ movl(frm, stk);

   618       __ subl(tmp, dat);

   619       __ movl(Operand(frmAddr()), tmp);

   621       // Align the stack to 16-bytes (each call adds 4 bytes).

   622       __ subl(esp, 12);

   623       break;

   625     case OP_IDXADDR_B:

   626     {

   627       cell_t val = readCell();

   628       __ shll(pri, val);

   629       __ addl(pri, alt);

   630       break;

   631     }

   633     case OP_SHL:

   634       __ movl(ecx, alt);

   635       __ shll_cl(pri);

   636       break;

   638     case OP_SHR:

   639       __ movl(ecx, alt);

   640       __ shrl_cl(pri);

   641       break;

   643     case OP_SSHR:

   644       __ movl(ecx, alt);

   645       __ sarl_cl(pri);

   646       break;

   648     case OP_SHL_C_PRI:

   649     case OP_SHL_C_ALT:

   650     {

   651       Register reg = (op == OP_SHL_C_PRI) ? pri : alt;

   652       cell_t val = readCell();

   653       __ shll(reg, val);

   654       break;

   655     }

   657     case OP_SHR_C_PRI:

   658     case OP_SHR_C_ALT:

   659     {

   660       Register reg = (op == OP_SHR_C_PRI) ? pri : alt;

   661       cell_t val = readCell();

   662       __ shrl(reg, val);

   663       break;

   664     }

   666     case OP_SMUL:

   667       __ imull(pri, alt);

   668       break;

   670     case OP_NOT:

   671       __ testl(eax, eax);

   672       __ movl(eax, 0);

   673       __ set(zero, r8_al);

   674       break;

   676     case OP_NEG:

   677       __ negl(eax);

   678       break;

   680     case OP_XOR:

   681       __ xorl(pri, alt);

   682       break;

   684     case OP_OR:

   685       __ orl(pri, alt);

   686       break;

   688     case OP_AND:

   689       __ andl(pri, alt);

   690       break;

   692     case OP_INVERT:

   693       __ notl(pri);

   694       break;

   696     case OP_ADD_C:

   697     {

   698       cell_t val = readCell();

   699       __ addl(pri, val);

   700       break;

   701     }

   703     case OP_SMUL_C:

   704     {

   705       cell_t val = readCell();

   706       __ imull(pri, pri, val);

   707       break;

   708     }

   710     case OP_EQ:

   711     case OP_NEQ:

   712     case OP_SLESS:

   713     case OP_SLEQ:

   714     case OP_SGRTR:

   715     case OP_SGEQ:

   716     {

   717       ConditionCode cc = OpToCondition(op);

   718       __ cmpl(pri, alt);

   719       __ movl(pri, 0);

   720       __ set(cc, r8_al);

   721       break;

   722     }

   724     case OP_EQ_C_PRI:

   725     case OP_EQ_C_ALT:

   726     {

   727       Register reg = (op == OP_EQ_C_PRI) ? pri : alt;

   728       cell_t val = readCell();

   729       __ cmpl(reg, val);

   730       __ movl(pri, 0);

   731       __ set(equal, r8_al);

   732       break;

   733     }

   735     case OP_INC_PRI:

   736     case OP_INC_ALT:

   737     {

   738       Register reg = (OP_INC_PRI) ? pri : alt;

   739       __ addl(reg, 1);

   740       break;

   741     }

   743     case OP_INC:

   744     case OP_INC_S:

   745     {

   746       Register base = (op == OP_INC) ? dat : frm;

   747       cell_t offset = readCell();

   748       __ addl(Operand(base, offset), 1);

   749       break;

   750     }

   752     case OP_INC_I:

   753       __ addl(Operand(dat, pri, NoScale), 1);

   754       break;

   756     case OP_DEC_PRI:

   757     case OP_DEC_ALT:

   758     {

   759       Register reg = (op == OP_DEC_PRI) ? pri : alt;

   760       __ subl(reg, 1);

   761       break;

   762     }

   764     case OP_DEC:

   765     case OP_DEC_S:

   766     {

   767       Register base = (op == OP_DEC) ? dat : frm;

   768       cell_t offset = readCell();

   769       __ subl(Operand(base, offset), 1);

   770       break;

   771     }

   773     case OP_DEC_I:

   774       __ subl(Operand(dat, pri, NoScale), 1);

   775       break;

   777     case OP_LOAD_PRI:

   778     case OP_LOAD_ALT:

   779     {

   780       Register reg = (op == OP_LOAD_PRI) ? pri : alt;

   781       cell_t offset = readCell();

   782       __ movl(reg, Operand(dat, offset));

   783       break;

   784     }

   786     case OP_LOAD_BOTH:

   787     {

   788       cell_t offs1 = readCell();

   789       cell_t offs2 = readCell();

   790       __ movl(pri, Operand(dat, offs1));

   791       __ movl(alt, Operand(dat, offs2));

   792       break;

   793     }

   795     case OP_LOAD_S_PRI:

   796     case OP_LOAD_S_ALT:

   797     {

   798       Register reg = (op == OP_LOAD_S_PRI) ? pri : alt;

   799       cell_t offset = readCell();

   800       __ movl(reg, Operand(frm, offset));

   801       break;

   802     }

   804     case OP_LOAD_S_BOTH:

   805     {

   806       cell_t offs1 = readCell();

   807       cell_t offs2 = readCell();

   808       __ movl(pri, Operand(frm, offs1));

   809       __ movl(alt, Operand(frm, offs2));

   810       break;

   811     }

   813     case OP_LREF_S_PRI:

   814     case OP_LREF_S_ALT:

   815     {

   816       Register reg = (op == OP_LREF_S_PRI) ? pri : alt;

   817       cell_t offset = readCell();

   818       __ movl(reg, Operand(frm, offset));

   819       __ movl(reg, Operand(dat, reg, NoScale));

   820       break;

   821     }

   823     case OP_CONST_PRI:

   824     case OP_CONST_ALT:

   825     {

   826       Register reg = (op == OP_CONST_PRI) ? pri : alt;

   827       cell_t val = readCell();

   828       __ movl(reg, val);

   829       break;

   830     }

   832     case OP_ADDR_PRI:

   833     case OP_ADDR_ALT:

   834     {

   835       Register reg = (op == OP_ADDR_PRI) ? pri : alt;

   836       cell_t offset = readCell();

   837       __ movl(reg, Operand(frmAddr()));

   838       __ addl(reg, offset);

   839       break;

   840     }

   842     case OP_STOR_PRI:

   843     case OP_STOR_ALT:

   844     {

   845       Register reg = (op == OP_STOR_PRI) ? pri : alt;

   846       cell_t offset = readCell();

   847       __ movl(Operand(dat, offset), reg);

   848       break;

   849     }

   851     case OP_STOR_S_PRI:

   852     case OP_STOR_S_ALT:

   853     {

   854       Register reg = (op == OP_STOR_S_PRI) ? pri : alt;

   855       cell_t offset = readCell();

   856       __ movl(Operand(frm, offset), reg);

   857       break;

   858     }

   860     case OP_IDXADDR:

   861       __ lea(pri, Operand(alt, pri, ScaleFour));

   862       break;

   864     case OP_SREF_S_PRI:

   865     case OP_SREF_S_ALT:

   866     {

   867       Register reg = (op == OP_SREF_S_PRI) ? pri : alt;

   868       cell_t offset = readCell();

   869       __ movl(tmp, Operand(frm, offset));

   870       __ movl(Operand(dat, tmp, NoScale), reg);

   871       break;

   872     }

   874     case OP_POP_PRI:

   875     case OP_POP_ALT:

   876     {

   877       Register reg = (op == OP_POP_PRI) ? pri : alt;

   878       __ movl(reg, Operand(stk, 0));

   879       __ addl(stk, 4);

   880       break;

   881     }

   883     case OP_SWAP_PRI:

   884     case OP_SWAP_ALT:

   885     {

   886       Register reg = (op == OP_SWAP_PRI) ? pri : alt;

   887       __ movl(tmp, Operand(stk, 0));

   888       __ movl(Operand(stk, 0), reg);

   889       __ movl(reg, tmp);

   890       break;

   891     }

   893     case OP_LIDX:

   894       __ lea(pri, Operand(alt, pri, ScaleFour));

   895       __ movl(pri, Operand(dat, pri, NoScale));

   896       break;

   898     case OP_LIDX_B:

   899     {

   900       cell_t val = readCell();

   901       if (val >= 0 && val <= 3) {

   902         __ lea(pri, Operand(alt, pri, Scale(val)));

   903       } else {

   904         __ shll(pri, val);

   905         __ addl(pri, alt);

   906       }

   907       emitCheckAddress(pri);

   908       __ movl(pri, Operand(dat, pri, NoScale));

   909       break;

   910     }

   912     case OP_CONST:

   913     case OP_CONST_S:

   914     {

   915       Register base = (op == OP_CONST) ? dat : frm;

   916       cell_t offset = readCell();

   917       cell_t val = readCell();

   918       __ movl(Operand(base, offset), val);

   919       break;

   920     }

   922     case OP_LOAD_I:

   923       emitCheckAddress(pri);

   924       __ movl(pri, Operand(dat, pri, NoScale));

   925       break;

   927     case OP_STOR_I:

   928       emitCheckAddress(alt);

   929       __ movl(Operand(dat, alt, NoScale), pri);

   930       break;

   932     case OP_SDIV:

   933     case OP_SDIV_ALT:

   934     {

   935       Register dividend = (op == OP_SDIV) ? pri : alt;

   936       Register divisor = (op == OP_SDIV) ? alt : pri;

   938       // Guard against divide-by-zero.

   939       __ testl(divisor, divisor);

   940       __ j(zero, &error_divide_by_zero_);

   942       // A more subtle case; -INT_MIN / -1 yields an overflow exception.

   943       Label ok;

   944       __ cmpl(divisor, -1);

   945       __ j(not_equal, &ok);

   946       __ cmpl(dividend, 0x80000000);

   947       __ j(equal, &error_integer_overflow_);

   948       __ bind(&ok);

   950       // Now we can actually perform the divide.

   951       __ movl(tmp, divisor);

   952       if (op == OP_SDIV)

   953         __ movl(edx, dividend);

   954       else

   955         __ movl(eax, dividend);

   956       __ sarl(edx, 31);

   957       __ idivl(tmp);

   958       break;

   959     }

   961     case OP_LODB_I:

   962     {

   963       cell_t val = readCell();

   964       emitCheckAddress(pri);

   965       __ movl(pri, Operand(dat, pri, NoScale));

   966       if (val == 1)

   967         __ andl(pri, 0xff);

   968       else if (val == 2)

   969         __ andl(pri, 0xffff);

   970       break;

   971     }

   973     case OP_STRB_I:

   974     {

   975       cell_t val = readCell();

   976       emitCheckAddress(alt);

   977       if (val == 1)

   978         __ movb(Operand(dat, alt, NoScale), pri);

   979       else if (val == 2)

   980         __ movw(Operand(dat, alt, NoScale), pri);

   981       else if (val == 4)

   982         __ movl(Operand(dat, alt, NoScale), pri);

   983       break;

   984     }

   986     case OP_RETN:

   987     {

   988       // Restore the old frame pointer.

   989       __ movl(frm, Operand(stk, 4));              // get the old frm

   990       __ addl(stk, 8);                            // pop stack

   991       __ movl(Operand(frmAddr()), frm);           // store back old frm

   992       __ addl(frm, dat);                          // relocate

   994       // Remove parameters.

   995       __ movl(tmp, Operand(stk, 0));

   996       __ lea(stk, Operand(stk, tmp, ScaleFour, 4));

   997       __ addl(esp, 12);

   998       __ ret();

   999       break;

  1000     }

  1002     case OP_MOVS:

  1003     {

  1004       cell_t val = readCell();

  1005       unsigned dwords = val / 4;

  1006       unsigned bytes = val % 4;

  1008       __ cld();

  1009       __ push(esi);

  1010       __ push(edi);

  1011       __ lea(esi, Operand(dat, pri, NoScale));

  1012       __ lea(edi, Operand(dat, alt, NoScale));

  1013       if (dwords) {

  1014         __ movl(ecx, dwords);

  1015         __ rep_movsd();

  1016       }

  1017       if (bytes) {

  1018         __ movl(ecx, bytes);

  1019         __ rep_movsb();

  1020       }

  1021       __ pop(edi);

  1022       __ pop(esi);

  1023       break;

  1024     }

  1026     case OP_FILL:

  1027     {

  1028       // eax/pri is used implicitly.

  1029       unsigned dwords = readCell() / 4;

  1030       __ push(edi);

  1031       __ lea(edi, Operand(dat, alt, NoScale));

  1032       __ movl(ecx, dwords);

  1033       __ cld();

  1034       __ rep_stosd();

  1035       __ pop(edi);

  1036       break;

  1037     }

  1039     case OP_STRADJUST_PRI:

  1040       __ addl(pri, 4);

  1041       __ sarl(pri, 2);

  1042       break;

  1044     case OP_FABS:

  1045       __ movl(pri, Operand(stk, 0));

  1046       __ andl(pri, 0x7fffffff);

  1047       __ addl(stk, 4);

  1048       break;

  1050     case OP_FLOAT:

  1051       if (MacroAssemblerX86::Features().sse2) {

  1052         __ cvtsi2ss(xmm0, Operand(edi, 0));

  1053         __ movd(pri, xmm0);

  1054       } else {

  1055         __ fild32(Operand(edi, 0));

  1056         __ subl(esp, 4);

  1057         __ fstp32(Operand(esp, 0));

  1058         __ pop(pri);

  1059       }

  1060       __ addl(stk, 4);

  1061       break;

  1063     case OP_FLOATADD:

  1064     case OP_FLOATSUB:

  1065     case OP_FLOATMUL:

  1066     case OP_FLOATDIV:

  1067       if (MacroAssemblerX86::Features().sse2) {

  1068         __ movss(xmm0, Operand(stk, 0));

  1069         if (op == OP_FLOATADD)

  1070           __ addss(xmm0, Operand(stk, 4));

  1071         else if (op == OP_FLOATSUB)

  1072           __ subss(xmm0, Operand(stk, 4));

  1073         else if (op == OP_FLOATMUL)

  1074           __ mulss(xmm0, Operand(stk, 4));

  1075         else if (op == OP_FLOATDIV)

  1076           __ divss(xmm0, Operand(stk, 4));

  1077         __ movd(pri, xmm0);

  1078       } else {

  1079         __ subl(esp, 4);

  1080         __ fld32(Operand(stk, 0));

  1082         if (op == OP_FLOATADD)

  1083           __ fadd32(Operand(stk, 4));

  1084         else if (op == OP_FLOATSUB)

  1085           __ fsub32(Operand(stk, 4));

  1086         else if (op == OP_FLOATMUL)

  1087           __ fmul32(Operand(stk, 4));

  1088         else if (op == OP_FLOATDIV)

  1089           __ fdiv32(Operand(stk, 4));

  1091         __ fstp32(Operand(esp, 0));

  1092         __ pop(pri);

  1093       }

  1094       __ addl(stk, 8);

  1095       break;

  1097     case OP_RND_TO_NEAREST:

  1098     {

  1099       if (MacroAssemblerX86::Features().sse) {

  1100         // Assume no one is touching MXCSR.

  1101         __ cvtss2si(pri, Operand(stk, 0));

  1102       } else {

  1103         static float kRoundToNearest = 0.5f;

  1104         // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1105         __ fld32(Operand(stk, 0));

  1106         __ fadd32(st0, st0);

  1107         __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));

  1108         __ subl(esp, 4);

  1109         __ fistp32(Operand(esp, 0));

  1110         __ pop(pri);

  1111         __ sarl(pri, 1);

  1112       }

  1113       __ addl(stk, 4);

  1114       break;

  1115     }

  1117     case OP_RND_TO_CEIL:

  1118     {

  1119       static float kRoundToCeil = -0.5f;

  1120       // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1121       __ fld32(Operand(stk, 0));

  1122       __ fadd32(st0, st0);

  1123       __ fsubr32(Operand(ExternalAddress(&kRoundToCeil)));

  1124       __ subl(esp, 4);

  1125       __ fistp32(Operand(esp, 0));

  1126       __ pop(pri);

  1127       __ sarl(pri, 1);

  1128       __ negl(pri);

  1129       __ addl(stk, 4);

  1130       break;

  1131     }

  1133     case OP_RND_TO_ZERO:

  1134       if (MacroAssemblerX86::Features().sse) {

  1135         __ cvttss2si(pri, Operand(stk, 0));

  1136       } else {

  1137         __ fld32(Operand(stk, 0));

  1138         __ subl(esp, 8);

  1139         __ fstcw(Operand(esp, 4));

  1140         __ movl(Operand(esp, 0), 0xfff);

  1141         __ fldcw(Operand(esp, 0));

  1142         __ fistp32(Operand(esp, 0));

  1143         __ pop(pri);

  1144         __ fldcw(Operand(esp, 0));

  1145         __ addl(esp, 4);

  1146       }

  1147       __ addl(stk, 4);

  1148       break;

  1150     case OP_RND_TO_FLOOR:

  1151       __ fld32(Operand(stk, 0));

  1152       __ subl(esp, 8);

  1153       __ fstcw(Operand(esp, 4));

  1154       __ movl(Operand(esp, 0), 0x7ff);

  1155       __ fldcw(Operand(esp, 0));

  1156       __ fistp32(Operand(esp, 0));

  1157       __ pop(eax);

  1158       __ fldcw(Operand(esp, 0));

  1159       __ addl(esp, 4);

  1160       __ addl(stk, 4);

  1161       break;

  1163     case OP_FLOATCMP:

  1164     {

  1165       Label bl, ab, done;

  1166       if (MacroAssemblerX86::Features().sse) {

  1167         __ movss(xmm0, Operand(stk, 4));

  1168         __ ucomiss(xmm0, Operand(stk, 0));

  1169       } else {

  1170         __ fld32(Operand(stk, 0));

  1171         __ fld32(Operand(stk, 4));

  1172         __ fucomip(st1);

  1173         __ fstp(st0);

  1174       }

  1175       __ j(above, &ab);

  1176       __ j(below, &bl);

  1177       __ xorl(pri, pri);

  1178       __ jmp(&done);

  1179       __ bind(&ab);

  1180       __ movl(pri, -1);

  1181       __ jmp(&done);

  1182       __ bind(&bl);

  1183       __ movl(pri, 1);

  1184       __ bind(&done);

  1185       __ addl(stk, 8);

  1186       break;

  1187     }

  1189     case OP_STACK:

  1190     {

  1191       cell_t amount = readCell();

  1192       __ addl(stk, amount);

  1194      if (amount > 0) {

  1195        // Check if the stack went beyond the stack top - usually a compiler error.

  1196        __ cmpl(stk, intptr_t(plugin_->memory + plugin_->mem_size));

  1197        __ j(not_below, &error_stack_min_);

  1198      } else {

  1199        // Check if the stack is going to collide with the heap.

  1200        __ movl(tmp, Operand(hpAddr()));

  1201        __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1202        __ cmpl(stk, tmp);

  1203        __ j(below, &error_stack_low_);

  1204      }

  1205      break;

  1206     }

  1208     case OP_HEAP:

  1209     {

  1210       cell_t amount = readCell();

  1211       __ movl(alt, Operand(hpAddr()));

  1212       __ addl(Operand(hpAddr()), amount);

  1214       if (amount > 0) {

  1215         __ cmpl(Operand(hpAddr()), plugin_->data_size);

  1216         __ j(below, &error_heap_min_);

  1217       } else {

  1218         __ movl(tmp, Operand(hpAddr()));

  1219         __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1220         __ cmpl(tmp, stk);

  1221         __ j(above, &error_heap_low_);

  1222       }

  1223       break;

  1224     }

  1226     case OP_JUMP:

  1227     {

  1228       Label *target = labelAt(readCell());

  1229       if (!target)

  1230         return false;

  1231       __ jmp(target);

  1232       break;

  1233     }

  1235     case OP_JZER:

  1236     case OP_JNZ:

  1237     {

  1238       ConditionCode cc = (op == OP_JZER) ? zero : not_zero;

  1239       Label *target = labelAt(readCell());

  1240       if (!target)

  1241         return false;

  1242       __ testl(pri, pri);

  1243       __ j(cc, target);

  1244       break;

  1245     }

  1247     case OP_JEQ:

  1248     case OP_JNEQ:

  1249     case OP_JSLESS:

  1250     case OP_JSLEQ:

  1251     case OP_JSGRTR:

  1252     case OP_JSGEQ:

  1253     {

  1254       Label *target = labelAt(readCell());

  1255       if (!target)

  1256         return false;

  1257       ConditionCode cc = OpToCondition(op);

  1258       __ cmpl(pri, alt);

  1259       __ j(cc, target);

  1260       break;

  1261     }

  1263     case OP_TRACKER_PUSH_C:

  1264     {

  1265       cell_t amount = readCell();

  1267       __ push(pri);

  1268       __ push(alt);

  1270       __ push(amount * 4);

  1271       __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));

  1272       __ call(ExternalAddress((void *)PushTracker));

  1273       __ addl(esp, 8);

  1274       __ testl(eax, eax);

  1275       __ j(not_zero, &extern_error_);

  1277       __ pop(alt);

  1278       __ pop(pri);

  1279       break;

  1280     }

  1282     case OP_TRACKER_POP_SETHEAP:

  1283     {

  1284       // Save registers.

  1285       __ push(pri);

  1286       __ push(alt);

  1288       // Get the context pointer and call the sanity checker.

  1289       __ push(intptr_t(rt_));

  1290       __ call(ExternalAddress((void *)PopTrackerAndSetHeap));

  1291       __ addl(esp, 4);

  1292       __ testl(eax, eax);

  1293       __ j(not_zero, &extern_error_);

  1295       __ pop(alt);

  1296       __ pop(pri);

  1297       break;

  1298     }

  1300     case OP_BREAK:

  1301     {

  1302       cell_t cip = uintptr_t(cip_ - 1) - uintptr_t(plugin_->pcode);

  1303       __ movl(Operand(cipAddr()), cip);

  1304       break;

  1305     }

  1307     case OP_HALT:

  1308       __ align(16);

  1309       __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1310       __ movl(Operand(tmp, offsetof(sp_context_t, rval)), pri);

  1311       __ movl(pri, readCell());

  1312       __ jmp(&extern_error_);

  1313       break;

  1315     case OP_BOUNDS:

  1316     {

  1317       cell_t value = readCell();

  1318       __ cmpl(eax, value);

  1319       __ j(above, &error_bounds_);

  1320       break;

  1321     }

  1323     case OP_GENARRAY:

  1324     case OP_GENARRAY_Z:

  1325       emitGenArray(op == OP_GENARRAY_Z);

  1326       break;

  1328     case OP_CALL:

  1329       if (!emitCall())

  1330         return false;

  1331       break;

  1333     case OP_SYSREQ_C:

  1334     case OP_SYSREQ_N:

  1335       if (!emitNativeCall(op))

  1336         return false;

  1337       break;

  1339     case OP_SWITCH:

  1340       if (!emitSwitch())

  1341         return false;

  1342       break;

  1344     case OP_CASETBL:

  1345     {

  1346       size_t ncases = readCell();

  1348       // Two cells per case, and one extra cell for the default address.

  1349       cip_ += (ncases * 2) + 1;

  1350       break;

  1351     }

  1353     case OP_NOP:

  1354       break;

  1356     default:

  1357       error_ = SP_ERROR_INVALID_INSTRUCTION;

  1358       return false;

  1359   }

  1361   return true;

  1362 }

  1364 Label *

  1365 Compiler::labelAt(size_t offset)

  1366 {

  1367   if (offset % 4 != 0 ||

  1368       offset > plugin_->pcode_size ||

  1369       offset <= pcode_start_)

  1370   {

  1371     // If the jump target is misaligned, or out of pcode bounds, or is an

  1372     // address out of the function bounds, we abort. Unfortunately we can't

  1373     // test beyond the end of the function since we don't have a precursor

  1374     // pass (yet).

  1375     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1376     return NULL;

  1377   }

  1379   return &jump_map_[offset / sizeof(cell_t)];

  1380 }

  1382 void

  1383 Compiler::emitCheckAddress(Register reg)

  1384 {

  1385   // Check if we're in memory bounds.

  1386   __ cmpl(reg, plugin_->mem_size);

  1387   __ j(not_below, &error_memaccess_);

  1389   // Check if we're in the invalid region between hp and sp.

  1390   Label done;

  1391   __ cmpl(reg, Operand(hpAddr()));

  1392   __ j(below, &done);

  1393   __ lea(tmp, Operand(dat, reg, NoScale));

  1394   __ cmpl(tmp, stk);

  1395   __ j(below, &error_memaccess_);

  1396   __ bind(&done);

  1397 }

  1399 void

  1400 Compiler::emitGenArray(bool autozero)

  1401 {

  1402   cell_t val = readCell();

  1403   if (val == 1)

  1404   {

  1405     // flat array; we can generate this without indirection tables.

  1406     // Note that we can overwrite ALT because technically STACK should be destroying ALT

  1407     __ movl(alt, Operand(hpAddr()));

  1408     __ movl(tmp, Operand(stk, 0));

  1409     __ movl(Operand(stk, 0), alt);    // store base of the array into the stack.

  1410     __ lea(alt, Operand(alt, tmp, ScaleFour));

  1411     __ movl(Operand(hpAddr()), alt);

  1412     __ addl(alt, dat);

  1413     __ cmpl(alt, stk);

  1414     __ j(not_below, &error_heap_low_);

  1416     __ shll(tmp, 2);

  1417     __ push(tmp);

  1418     __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));

  1419     __ call(ExternalAddress((void *)PushTracker));

  1420     __ addl(esp, 4);

  1421     __ pop(tmp);

  1422     __ shrl(tmp, 2);

  1423     __ testl(eax, eax);

  1424     __ j(not_zero, &extern_error_);

  1426     if (autozero) {

  1427       // Note - tmp is ecx and still intact.

  1428       __ push(eax);

  1429       __ push(edi);

  1430       __ xorl(eax, eax);

  1431       __ movl(edi, Operand(stk, 0));

  1432       __ addl(edi, dat);

  1433       __ cld();

  1434       __ rep_stosd();

  1435       __ pop(edi);

  1436       __ pop(eax);

  1437     }

  1438   } else {

  1439     __ push(pri);

  1441     // int GenerateArray(sp_plugin_t, vars[], uint32_t, cell_t *, int, unsigned *);

  1442     __ push(autozero ? 1 : 0);

  1443     __ push(stk);

  1444     __ push(val);

  1445     __ push(intptr_t(rt_));

  1446     __ call(ExternalAddress((void *)GenerateArray));

  1447     __ addl(esp, 4 * sizeof(void *));

  1449     // restore pri to tmp

  1450     __ pop(tmp);

  1452     __ testl(eax, eax);

  1453     __ j(not_zero, &extern_error_);

  1455     // Move tmp back to pri, remove pushed args.

  1456     __ movl(pri, tmp);

  1457     __ addl(stk, (val - 1) * 4);

  1458   }

  1459 }

  1461 bool

  1462 Compiler::emitCall()

  1463 {

  1464   cell_t offset = readCell();

  1466   // If this offset looks crappy, i.e. not aligned or out of bounds, we just

  1467   // abort.

  1468   if (offset % 4 != 0 || uint32_t(offset) >= plugin_->pcode_size) {

  1469     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1470     return false;

  1471   }

  1473   // eax = context

  1474   // ecx = rp

  1475   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1476   __ movl(ecx, Operand(eax, offsetof(sp_context_t, rp)));

  1478   // Check if the return stack is used up.

  1479   __ cmpl(ecx, SP_MAX_RETURN_STACK);

  1480   __ j(not_below, &error_stack_low_);

  1482   // Add to the return stack.

  1483   uintptr_t cip = uintptr_t(cip_ - 2) - uintptr_t(plugin_->pcode);

  1484   __ movl(Operand(eax, ecx, ScaleFour, offsetof(sp_context_t, rstk_cips)), cip);

  1486   // Increment the return stack pointer.

  1487   __ addl(Operand(eax, offsetof(sp_context_t, rp)), 1);

  1489   // Store the CIP of the function we're about to call.

  1490   __ movl(Operand(cipAddr()), offset);

  1492   JitFunction *fun = rt_->GetJittedFunctionByOffset(offset);

  1493   if (!fun) {

  1494     // Need to emit a delayed thunk.

  1495     CallThunk *thunk = new CallThunk(offset);

  1496     __ call(&thunk->call);

  1497     if (!thunks_.append(thunk))

  1498       return false;

  1499   } else {

  1500     // Function is already emitted, we can do a direct call.

  1501     __ call(ExternalAddress(fun->GetEntryAddress()));

  1502   }

  1504   // Restore the last cip.

  1505   __ movl(Operand(cipAddr()), cip);

  1507   // Mark us as leaving the last frame.

  1508   __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1509   __ subl(Operand(tmp, offsetof(sp_context_t, rp)), 1);

  1510   return true;

  1511 }

  1513 void

  1514 Compiler::emitCallThunks()

  1515 {

  1516   for (size_t i = 0; i < thunks_.length(); i++) {

  1517     CallThunk *thunk = thunks_[i];

  1519     Label error;

  1520     __ bind(&thunk->call);

  1521     // Huge hack - get the return address, since that is the call that we

  1522     // need to patch.

  1523     __ movl(eax, Operand(esp, 0));

  1525     // Align the stack. Unfortunately this is a pretty careful dance based

  1526     // on the number of words we push (4 args, 1 saved esp), and someday

  1527     // we should look into automating this.

  1528     __ movl(edx, esp);

  1529     __ andl(esp, 0xfffffff0);

  1530     __ subl(esp, 12);

  1531     __ push(edx);

  1533     // Push arguments.

  1534     __ push(eax);

  1535     __ subl(esp, 4);

  1536     __ movl(Operand(esp, 0), esp);

  1537     __ push(thunk->pcode_offset);

  1538     __ push(intptr_t(rt_));

  1539     __ call(ExternalAddress((void *)CompileFromThunk));

  1540     __ movl(edx, Operand(esp, 8));

  1541     __ movl(esp, Operand(esp, 16));

  1542     __ testl(eax, eax);

  1543     __ j(not_zero, &error);

  1544     __ call(edx);

  1545     __ ret();

  1547     __ bind(&error);

  1548     __ movl(Operand(cipAddr()), thunk->pcode_offset);

  1549     __ jmp(g_Jit.GetUniversalReturn());

  1550   }

  1551 }

  1553 cell_t

  1554 Compiler::readCell()

  1555 {

  1556   if (cip_ >= code_end_) {

  1557     error_= SP_ERROR_INVALID_INSTRUCTION;

  1558     return 0;

  1559   }

  1560   return *cip_++;

  1561 }

  1563 bool

  1564 Compiler::emitNativeCall(OPCODE op)

  1565 {

  1566   uint32_t native_index = readCell();

  1568   if (native_index >= plugin_->num_natives) {

  1569     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1570     return false;

  1571   }

  1573   uint32_t num_params;

  1574   if (op == OP_SYSREQ_N) {

  1575     num_params = readCell();

  1577     // See if we can get a replacement opcode. If we can, then recursively

  1578     // call emitOp() to generate it. Note: it's important that we do this

  1579     // before generating any code for the SYSREQ.N.

  1580     unsigned replacement = rt_->GetNativeReplacement(native_index);

  1581     if (replacement != OP_NOP)

  1582       return emitOp((OPCODE)replacement);

  1584     // Store the number of parameters.

  1585     __ movl(Operand(stk, -4), num_params);

  1586     __ subl(stk, 4);

  1587   }

  1589   // Save registers.

  1590   __ push(edx);

  1592   // Push the parameters for the C++ function.

  1593   __ push(stk);

  1594   __ push(native_index);

  1596   // Relocate our absolute stk to be dat-relative, and update the context's

  1597   // view.

  1598   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1599   __ subl(stk, dat);

  1600   __ movl(Operand(eax, offsetof(sp_context_t, sp)), stk);

  1602   // Push context and make the call.

  1603   __ push(eax);

  1604   __ call(ExternalAddress((void *)NativeCallback));

  1606   // Check for errors.

  1607   __ movl(ecx, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1608   __ movl(ecx, Operand(ecx, offsetof(sp_context_t, n_err)));

  1609   __ testl(ecx, ecx);

  1610   __ j(not_zero, &extern_error_);

  1612   // Restore local state.

  1613   __ addl(stk, dat);

  1614   __ addl(esp, 12);

  1615   __ pop(edx);

  1617   if (op == OP_SYSREQ_N) {

  1618     // Pop the stack. Do not check the margins.

  1619     __ addl(stk, (num_params + 1) * sizeof(cell_t));

  1620   }

  1621   return true;

  1622 }

  1624 bool

  1625 Compiler::emitSwitch()

  1626 {

  1627   cell_t offset = readCell();

  1628   if (!labelAt(offset))

  1629     return false;

  1631   cell_t *tbl = (cell_t *)((char *)plugin_->pcode + offset + sizeof(cell_t));

  1633   struct Entry {

  1634     cell_t val;

  1635     cell_t offset;

  1636   };

  1638   size_t ncases = *tbl++;

  1640   Label *defaultCase = labelAt(*tbl);

  1641   if (!defaultCase)

  1642     return false;

  1644   // Degenerate - 0 cases.

  1645   if (!ncases) {

  1646     __ jmp(defaultCase);

  1647     return true;

  1648   }

  1650   Entry *cases = (Entry *)(tbl + 1);

  1652   // Degenerate - 1 case.

  1653   if (ncases == 1) {

  1654     Label *maybe = labelAt(cases[0].offset);

  1655     if (!maybe)

  1656       return false;

  1657     __ cmpl(pri, cases[0].val);

  1658     __ j(equal, maybe);

  1659     __ jmp(defaultCase);

  1660     return true;

  1661   }

  1663   // We have two or more cases, so let's generate a full switch. Decide

  1664   // whether we'll make an if chain, or a jump table, based on whether

  1665   // the numbers are strictly sequential.

  1666   bool sequential = true;

  1667   {

  1668     cell_t first = cases[0].val;

  1669     cell_t last = first;

  1670     for (size_t i = 1; i < ncases; i++) {

  1671       if (cases[i].val != ++last) {

  1672         sequential = false;

  1673         break;

  1674       }

  1675     }

  1676   }

  1678   // First check whether the bounds are correct: if (a < LOW || a > HIGH);

  1679   // this check is valid whether or not we emit a sequential-optimized switch.

  1680   cell_t low = cases[0].val;

  1681   if (low != 0) {

  1682     // negate it so we'll get a lower bound of 0.

  1683     low = -low;

  1684     __ lea(tmp, Operand(pri, low));

  1685   } else {

  1686     __ movl(tmp, pri);

  1687   }

  1689   cell_t high = abs(cases[0].val - cases[ncases - 1].val);

  1690   __ cmpl(tmp, high);

  1691   __ j(above, defaultCase);

  1693   if (sequential) {

  1694     // Optimized table version. The tomfoolery below is because we only have

  1695     // one free register... it seems unlikely pri or alt will be used given

  1696     // that we're at the end of a control-flow point, but we'll play it safe.

  1697     DataLabel table;

  1698     __ push(eax);

  1699     __ movl(eax, &table);

  1700     __ movl(ecx, Operand(eax, ecx, ScaleFour));

  1701     __ pop(eax);

  1702     __ jmp(ecx);

  1704     __ bind(&table);

  1705     for (size_t i = 0; i < ncases; i++) {

  1706       Label *label = labelAt(cases[i].offset);

  1707       if (!label)

  1708         return false;

  1709       __ emit_absolute_address(label);

  1710     }

  1711   } else {

  1712     // Slower version. Go through each case and generate a check.

  1713     for (size_t i = 0; i < ncases; i++) {

  1714       Label *label = labelAt(cases[i].offset);

  1715       if (!label)

  1716         return false;

  1717       __ cmpl(pri, cases[i].val);

  1718       __ j(equal, label);

  1719     }

  1720     __ jmp(defaultCase);

  1721   }

  1722   return true;

  1723 }

  1725 void

  1726 Compiler::emitErrorPath(Label *dest, int code)

  1727 {

  1728   if (dest->used()) {

  1729     __ bind(dest);

  1730     __ movl(eax, code);

  1731     __ jmp(g_Jit.GetUniversalReturn());

  1732   }

  1733 }

  1735 void

  1736 Compiler::emitErrorPaths()

  1737 {

  1738   emitErrorPath(&error_divide_by_zero_, SP_ERROR_DIVIDE_BY_ZERO);

  1739   emitErrorPath(&error_stack_low_, SP_ERROR_STACKLOW);

  1740   emitErrorPath(&error_stack_min_, SP_ERROR_STACKMIN);

  1741   emitErrorPath(&error_bounds_, SP_ERROR_ARRAY_BOUNDS);

  1742   emitErrorPath(&error_memaccess_, SP_ERROR_MEMACCESS);

  1743   emitErrorPath(&error_heap_low_, SP_ERROR_HEAPLOW);

  1744   emitErrorPath(&error_heap_min_, SP_ERROR_HEAPMIN);

  1745   emitErrorPath(&error_integer_overflow_, SP_ERROR_INTEGER_OVERFLOW);

  1747   if (extern_error_.used()) {

  1748     __ bind(&extern_error_);

  1749     __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1750     __ movl(eax, Operand(eax, offsetof(sp_context_t, n_err)));

  1751     __ jmp(g_Jit.GetUniversalReturn());

  1752   }

  1753 }

  1755 typedef int (*JIT_EXECUTE)(sp_context_t *ctx, uint8_t *memory, void *code);

  1757 static void *

  1758 GenerateEntry(void **retp)

  1759 {

  1760   AssemblerX86 masm;

  1762   __ push(ebp);

  1763   __ movl(ebp, esp);

  1765   __ push(esi);   // ebp - 4

  1766   __ push(edi);   // ebp - 8

  1767   __ push(ebx);   // ebp - 12

  1768   __ push(esp);   // ebp - 16

  1770   __ movl(ebx, Operand(ebp, 8 + 4 * 0));

  1771   __ movl(eax, Operand(ebp, 8 + 4 * 1));

  1772   __ movl(ecx, Operand(ebp, 8 + 4 * 2));

  1774   // Set up run-time registers.

  1775   __ movl(edi, Operand(ebx, offsetof(sp_context_t, sp)));

  1776   __ addl(edi, eax);

  1777   __ movl(esi, eax);

  1778   __ movl(ebx, edi);

  1780   // Align the stack.

  1781   __ andl(esp, 0xfffffff0);

  1783   // Call into plugin (align the stack first).

  1784   __ call(ecx);

  1786   // Get input context, store rval.

  1787   __ movl(ecx, Operand(ebp, 8 + 4 * 0));

  1788   __ movl(Operand(ecx, offsetof(sp_context_t, rval)), pri);

  1790   // Set no error.

  1791   __ movl(eax, SP_ERROR_NONE);

  1793   // Store latest stk. If we have an error code, we'll jump directly to here,

  1794   // so eax will already be set.

  1795   Label ret;

  1796   __ bind(&ret);

  1797   __ subl(stk, dat);

  1798   __ movl(Operand(ecx, offsetof(sp_context_t, sp)), stk);

  1800   // Restore stack.

  1801   __ movl(esp, Operand(ebp, -16));

  1803   // Restore registers and gtfo.

  1804   __ pop(ebx);

  1805   __ pop(edi);

  1806   __ pop(esi);

  1807   __ pop(ebp);

  1808   __ ret();

  1810   // The universal emergency return will jump to here.

  1811   Label error;

  1812   __ bind(&error);

  1813   __ movl(ecx, Operand(ebp, 8 + 4 * 0)); // ret-path expects ecx = ctx

  1814   __ jmp(&ret);

  1816   void *code = LinkCode(masm);

  1817   if (!code)

  1818     return NULL;

  1820   *retp = reinterpret_cast<uint8_t *>(code) + error.offset();

  1821   return code;

  1822 }

  1824 ICompilation *JITX86::ApplyOptions(ICompilation *_IN, ICompilation *_OUT)

  1825 {

  1826   if (_IN == NULL)

  1827     return _OUT;

  1829   CompData *_in = (CompData * )_IN;

  1830   CompData *_out = (CompData * )_OUT;

  1832   _in->inline_level = _out->inline_level;

  1833   _in->profile = _out->profile;

  1835   _out->Abort();

  1836   return _in;

  1837 }

  1839 JITX86::JITX86()

  1840 {

  1841   m_pJitEntry = NULL;

  1842 }

  1844 bool JITX86::InitializeJIT()

  1845 {

  1846   g_pCodeCache = KE_CreateCodeCache();

  1848   m_pJitEntry = GenerateEntry(&m_pJitReturn);

  1849   if (!m_pJitEntry)

  1850     return false;

  1852   MacroAssemblerX86 masm;

  1853   MacroAssemblerX86::GenerateFeatureDetection(masm);

  1854   void *code = LinkCode(masm);

  1855   if (!code)

  1856     return false;

  1857   MacroAssemblerX86::RunFeatureDetection(code);

  1858   KE_FreeCode(g_pCodeCache, code);

  1860   return true;

  1861 }

  1863 void

  1864 JITX86::ShutdownJIT()

  1865 {

  1866   KE_DestroyCodeCache(g_pCodeCache);

  1867 }

  1869 JitFunction *

  1870 JITX86::CompileFunction(BaseRuntime *prt, cell_t pcode_offs, int *err)

  1871 {

  1872   Compiler cc(prt, pcode_offs);

  1873   JitFunction *fun = cc.emit(err);

  1874   if (!fun)

  1875     return NULL;

  1876   prt->AddJittedFunction(fun);

  1877   return fun;

  1878 }

  1880 void

  1881 JITX86::SetupContextVars(BaseRuntime *runtime, BaseContext *pCtx, sp_context_t *ctx)

  1882 {

  1883   tracker_t *trk = new tracker_t;

  1885   ctx->vm[JITVARS_TRACKER] = trk;

  1886   ctx->vm[JITVARS_BASECTX] = pCtx; /* GetDefaultContext() is not constructed yet */

  1887   ctx->vm[JITVARS_PROFILER] = g_engine2.GetProfiler();

  1888   ctx->vm[JITVARS_PLUGIN] = const_cast<sp_plugin_t *>(runtime->plugin());

  1890   trk->pBase = (ucell_t *)malloc(1024);

  1891   trk->pCur = trk->pBase;

  1892   trk->size = 1024 / sizeof(cell_t);

  1893 }

  1895 SPVM_NATIVE_FUNC

  1896 JITX86::CreateFakeNative(SPVM_FAKENATIVE_FUNC callback, void *pData)

  1897 {

  1898   AssemblerX86 masm;

  1900   __ push(ebx);

  1901   __ push(edi);

  1902   __ push(esi);

  1903   __ movl(edi, Operand(esp, 16)); // store ctx

  1904   __ movl(esi, Operand(esp, 20)); // store params

  1905   __ movl(ebx, esp);

  1906   __ andl(esp, 0xfffffff0);

  1907   __ subl(esp, 4);

  1909   __ push(intptr_t(pData));

  1910   __ push(esi);

  1911   __ push(edi);

  1912   __ call(ExternalAddress((void *)callback));

  1913   __ movl(esp, ebx);

  1914   __ pop(esi);

  1915   __ pop(edi);

  1916   __ pop(ebx);

  1917   __ ret();

  1919   return (SPVM_NATIVE_FUNC)LinkCode(masm);

  1920 }

  1922 void JITX86::DestroyFakeNative(SPVM_NATIVE_FUNC func)

  1923 {

  1924   KE_FreeCode(g_pCodeCache, (void *)func);

  1925 }

  1927 ICompilation *JITX86::StartCompilation()

  1928 {

  1929   return new CompData;

  1930 }

  1932 ICompilation *JITX86::StartCompilation(BaseRuntime *runtime)

  1933 {

  1934   return new CompData;

  1935 }

  1937 void CompData::Abort()

  1938 {

  1939   delete this;

  1940 }

  1942 void JITX86::FreeContextVars(sp_context_t *ctx)

  1943 {

  1944   free(((tracker_t *)(ctx->vm[JITVARS_TRACKER]))->pBase);

  1945   delete (tracker_t *)ctx->vm[JITVARS_TRACKER];

  1946 }

  1948 bool CompData::SetOption(const char *key, const char *val)

  1949 {

  1950   if (strcmp(key, SP_JITCONF_DEBUG) == 0)

  1951     return true;

  1952   if (strcmp(key, SP_JITCONF_PROFILE) == 0) {

  1953     profile = atoi(val);

  1955     /** Callbacks must be profiled to profile functions! */

  1956     if ((profile & SP_PROF_FUNCTIONS) == SP_PROF_FUNCTIONS)

  1957       profile |= SP_PROF_CALLBACKS;

  1959     return true;

  1960   }

  1962   return false;

  1963 }

  1965 int JITX86::InvokeFunction(BaseRuntime *runtime, JitFunction *fn, cell_t *result)

  1966 {

  1967   sp_context_t *ctx = runtime->GetBaseContext()->GetCtx();

  1969   // Note that cip, hp, sp are saved and restored by Execute2().

  1970   ctx->cip = fn->GetPCodeAddress();

  1972   JIT_EXECUTE pfn = (JIT_EXECUTE)m_pJitEntry;

  1973   int err = pfn(ctx, runtime->plugin()->memory, fn->GetEntryAddress());

  1975   *result = ctx->rval;

  1976   return err;

  1977 }

  1979 void *JITX86::AllocCode(size_t size)

  1980 {

  1981   return Knight::KE_AllocCode(g_pCodeCache, size);

  1982 }

  1984 void JITX86::FreeCode(void *code)

  1985 {

  1986   KE_FreeCode(g_pCodeCache, code);

  1987 }

author	David Anderson <dvander@alliedmods.net>
	Mon Aug 12 00:24:02 2013 +0100 (2013-08-12)
changeset 3993	d06d6dacfa18
parent 3992	f1e8cad30a64
child 4013	6201c1180e0e
permissions	-rw-r--r--