Home - sourcemod-central - file revision - sourcepawn/jit/x86/jit_x86.cpp@3e58428d988f Mercurial
summary | shortlog | changelog | graph | tags | branches | files | changeset | file | latest | revisions | annotate | diff | raw | help
sourcepawn/jit/x86/jit_x86.cpp
author David Anderson <dvander@alliedmods.net>
Sun Sep 01 00:23:44 2013 -0700 (2013-09-01)
changeset 4116 3e58428d988f
parent 4100 10bc39fcb710
child 4327 60e4042f2589
permissions -rw-r--r--
Add an interpreter because. (bug 5902, r=ds). 
     1 /**
     2  * vim: set ts=2 sw=2 tw=99 et:
     3  * =============================================================================
     4  * SourceMod
     5  * Copyright (C) 2004-2008 AlliedModders LLC.  All rights reserved.
     6  * =============================================================================
     7  *
     8  * This program is free software; you can redistribute it and/or modify it under
     9  * the terms of the GNU General Public License, version 3.0, as published by the
    10  * Free Software Foundation.
    11  * 
    12  * This program is distributed in the hope that it will be useful, but WITHOUT
    13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    14  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    15  * details.
    16  *
    17  * You should have received a copy of the GNU General Public License along with
    18  * this program.  If not, see <http://www.gnu.org/licenses/>.
    19  *
    20  * As a special exception, AlliedModders LLC gives you permission to link the
    21  * code of this program (as well as its derivative works) to "Half-Life 2," the
    22  * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
    23  * by the Valve Corporation.  You must obey the GNU General Public License in
    24  * all respects for all other code used.  Additionally, AlliedModders LLC grants
    25  * this exception to all derivative works.  AlliedModders LLC defines further
    26  * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
    27  * or <http://www.sourcemod.net/license.php>.
    28  *
    29  * Version: $Id$
    30  */
    31 
    32 #include <string.h>
    33 #include <stdlib.h>
    34 #include <assert.h>
    35 #include "jit_x86.h"
    36 #include "../sp_vm_engine.h"
    37 #include "../engine2.h"
    38 #include "../BaseRuntime.h"
    39 #include "../sp_vm_basecontext.h"
    40 #include "watchdog_timer.h"
    41 #include "interpreter.h"
    42 
    43 using namespace Knight;
    44 
    45 #if defined USE_UNGEN_OPCODES
    46 #include "ungen_opcodes.h"
    47 #endif
    48 
    49 #define __ masm.
    50 
    51 JITX86 g_Jit;
    52 KeCodeCache *g_pCodeCache = NULL;
    53 ISourcePawnEngine *engine = &g_engine1;
    54 
    55 static inline uint8_t *
    56 LinkCode(AssemblerX86 &masm)
    57 {
    58   if (masm.outOfMemory())
    59     return NULL;
    60 
    61   void *code = Knight::KE_AllocCode(g_pCodeCache, masm.length());
    62   if (!code)
    63     return NULL;
    64 
    65   masm.emitToExecutableMemory(code);
    66   return reinterpret_cast<uint8_t *>(code);
    67 }
    68 
    69 static inline ConditionCode
    70 OpToCondition(OPCODE op)
    71 {
    72   switch (op) {
    73    case OP_EQ:
    74    case OP_JEQ:
    75     return equal;
    76    case OP_NEQ:
    77    case OP_JNEQ:
    78     return not_equal;
    79    case OP_SLESS:
    80    case OP_JSLESS:
    81     return less;
    82    case OP_SLEQ:
    83    case OP_JSLEQ:
    84     return less_equal;
    85    case OP_SGRTR:
    86    case OP_JSGRTR:
    87     return greater;
    88    case OP_SGEQ:
    89    case OP_JSGEQ:
    90     return greater_equal;
    91    default:
    92     assert(false);
    93     return negative;
    94   }
    95 }
    96 
    97 struct array_creation_t
    98 {
    99   const cell_t *dim_list;     /* Dimension sizes */
   100   cell_t dim_count;           /* Number of dimensions */
   101   cell_t *data_offs;          /* Current offset AFTER the indirection vectors (data) */
   102   cell_t *base;               /* array base */
   103 };
   104 
   105 static cell_t
   106 GenerateInnerArrayIndirectionVectors(array_creation_t *ar, int dim, cell_t cur_offs)
   107 {
   108   cell_t write_offs = cur_offs;
   109   cell_t *data_offs = ar->data_offs;
   110 
   111   cur_offs += ar->dim_list[dim];
   112 
   113   // Dimension n-x where x > 2 will have sub-vectors.  
   114   // Otherwise, we just need to reference the data section.
   115   if (ar->dim_count > 2 && dim < ar->dim_count - 2) {
   116     // For each index at this dimension, write offstes to our sub-vectors.
   117     // After we write one sub-vector, we generate its sub-vectors recursively.
   118     // At the end, we're given the next offset we can use.
   119     for (int i = 0; i < ar->dim_list[dim]; i++) {
   120       ar->base[write_offs] = (cur_offs - write_offs) * sizeof(cell_t);
   121       write_offs++;
   122       cur_offs = GenerateInnerArrayIndirectionVectors(ar, dim + 1, cur_offs);
   123     }
   124   } else {
   125     // In this section, there are no sub-vectors, we need to write offsets 
   126     // to the data.  This is separate so the data stays in one big chunk.
   127     // The data offset will increment by the size of the last dimension, 
   128     // because that is where the data is finally computed as. 
   129     for (int i = 0; i < ar->dim_list[dim]; i++) {
   130       ar->base[write_offs] = (*data_offs - write_offs) * sizeof(cell_t);
   131       write_offs++;
   132       *data_offs = *data_offs + ar->dim_list[dim + 1];
   133     }
   134   }
   135 
   136   return cur_offs;
   137 }
   138 
   139 static cell_t
   140 calc_indirection(const array_creation_t *ar, cell_t dim)
   141 {
   142   cell_t size = ar->dim_list[dim];
   143   if (dim < ar->dim_count - 2)
   144     size += ar->dim_list[dim] * calc_indirection(ar, dim + 1);
   145   return size;
   146 }
   147 
   148 static cell_t
   149 GenerateArrayIndirectionVectors(cell_t *arraybase, cell_t dims[], cell_t _dimcount, bool autozero)
   150 {
   151   array_creation_t ar;
   152   cell_t data_offs;
   153 
   154   /* Reverse the dimensions */
   155   cell_t dim_list[sDIMEN_MAX];
   156   int cur_dim = 0;
   157   for (int i = _dimcount - 1; i >= 0; i--)
   158     dim_list[cur_dim++] = dims[i];
   159   
   160   ar.base = arraybase;
   161   ar.dim_list = dim_list;
   162   ar.dim_count = _dimcount;
   163   ar.data_offs = &data_offs;
   164 
   165   data_offs = calc_indirection(&ar, 0);
   166   GenerateInnerArrayIndirectionVectors(&ar, 0, 0);
   167   return data_offs;
   168 }
   169 
   170 int
   171 GenerateFullArray(BaseRuntime *rt, uint32_t argc, cell_t *argv, int autozero)
   172 {
   173   sp_context_t *ctx = rt->GetBaseContext()->GetCtx();
   174 
   175   // Calculate how many cells are needed.
   176   if (argv[0] <= 0)
   177     return SP_ERROR_ARRAY_TOO_BIG;
   178 
   179   uint32_t cells = argv[0];
   180 
   181   for (uint32_t dim = 1; dim < argc; dim++) {
   182     cell_t dimsize = argv[dim];
   183     if (dimsize <= 0)
   184       return SP_ERROR_ARRAY_TOO_BIG;
   185     if (!ke::IsUint32MultiplySafe(cells, dimsize))
   186       return SP_ERROR_ARRAY_TOO_BIG;
   187     cells *= uint32_t(dimsize);
   188     if (!ke::IsUint32AddSafe(cells, dimsize))
   189       return SP_ERROR_ARRAY_TOO_BIG;
   190     cells += uint32_t(dimsize);
   191   }
   192 
   193   if (!ke::IsUint32MultiplySafe(cells, 4))
   194     return SP_ERROR_ARRAY_TOO_BIG;
   195 
   196   uint32_t bytes = cells * 4;
   197   if (!ke::IsUint32AddSafe(ctx->hp, bytes))
   198     return SP_ERROR_ARRAY_TOO_BIG;
   199 
   200   uint32_t new_hp = ctx->hp + bytes;
   201   cell_t *dat_hp = reinterpret_cast<cell_t *>(rt->plugin()->memory + new_hp);
   202 
   203   // argv, coincidentally, is STK.
   204   if (dat_hp >= argv - STACK_MARGIN)
   205     return SP_ERROR_HEAPLOW;
   206 
   207   if (int err = PushTracker(rt->GetBaseContext()->GetCtx(), bytes))
   208     return err;
   209 
   210   cell_t *base = reinterpret_cast<cell_t *>(rt->plugin()->memory + ctx->hp);
   211   cell_t offs = GenerateArrayIndirectionVectors(base, argv, argc, !!autozero);
   212   assert(size_t(offs) == cells);
   213 
   214   argv[argc - 1] = ctx->hp;
   215   ctx->hp = new_hp;
   216   return SP_ERROR_NONE;
   217 }
   218 
   219 #if !defined NDEBUG
   220 static const char *
   221 GetFunctionName(const sp_plugin_t *plugin, uint32_t offs)
   222 {
   223   if (!plugin->debug.unpacked) {
   224     uint32_t max, iter;
   225     sp_fdbg_symbol_t *sym;
   226     sp_fdbg_arraydim_t *arr;
   227     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);
   228 
   229     max = plugin->debug.syms_num;
   230     for (iter = 0; iter < max; iter++) {
   231       sym = (sp_fdbg_symbol_t *)cursor;
   232 
   233       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)
   234         return plugin->debug.stringbase + sym->name;
   235 
   236       if (sym->dimcount > 0) {
   237         cursor += sizeof(sp_fdbg_symbol_t);
   238         arr = (sp_fdbg_arraydim_t *)cursor;
   239         cursor += sizeof(sp_fdbg_arraydim_t) * sym->dimcount;
   240         continue;
   241       }
   242 
   243       cursor += sizeof(sp_fdbg_symbol_t);
   244     }
   245   } else {
   246     uint32_t max, iter;
   247     sp_u_fdbg_symbol_t *sym;
   248     sp_u_fdbg_arraydim_t *arr;
   249     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);
   250 
   251     max = plugin->debug.syms_num;
   252     for (iter = 0; iter < max; iter++) {
   253       sym = (sp_u_fdbg_symbol_t *)cursor;
   254 
   255       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)
   256         return plugin->debug.stringbase + sym->name;
   257 
   258       if (sym->dimcount > 0) {
   259         cursor += sizeof(sp_u_fdbg_symbol_t);
   260         arr = (sp_u_fdbg_arraydim_t *)cursor;
   261         cursor += sizeof(sp_u_fdbg_arraydim_t) * sym->dimcount;
   262         continue;
   263       }
   264 
   265       cursor += sizeof(sp_u_fdbg_symbol_t);
   266     }
   267   }
   268 
   269   return NULL;
   270 }
   271 #endif
   272 
   273 static int
   274 CompileFromThunk(BaseRuntime *runtime, cell_t pcode_offs, void **addrp, char *pc)
   275 {
   276   // If the watchdog timer has declared a timeout, we must process it now,
   277   // and possibly refuse to compile, since otherwise we will compile a
   278   // function that is not patched for timeouts.
   279   if (!g_WatchdogTimer.HandleInterrupt())
   280     return SP_ERROR_TIMEOUT;
   281 
   282   JitFunction *fn = runtime->GetJittedFunctionByOffset(pcode_offs);
   283   if (!fn) {
   284     int err;
   285     fn = g_Jit.CompileFunction(runtime, pcode_offs, &err);
   286     if (!fn)
   287       return err;
   288   }
   289 
   290 #if !defined NDEBUG
   291   g_engine1.GetDebugHook()->OnDebugSpew(
   292       "Patching thunk to %s::%s\n",
   293       runtime->plugin()->name,
   294       GetFunctionName(runtime->plugin(), pcode_offs));
   295 #endif
   296 
   297   *addrp = fn->GetEntryAddress();
   298 
   299   /* Right now, we always keep the code RWE */
   300   *(intptr_t *)(pc - 4) = intptr_t(fn->GetEntryAddress()) - intptr_t(pc);
   301   return SP_ERROR_NONE;
   302 }
   303 
   304 Compiler::Compiler(BaseRuntime *rt, cell_t pcode_offs)
   305   : rt_(rt),
   306     plugin_(rt->plugin()),
   307     error_(SP_ERROR_NONE),
   308     pcode_start_(pcode_offs),
   309     code_start_(reinterpret_cast<cell_t *>(plugin_->pcode + pcode_start_)),
   310     cip_(code_start_),
   311     code_end_(reinterpret_cast<cell_t *>(plugin_->pcode + plugin_->pcode_size))
   312 {
   313   size_t nmaxops = plugin_->pcode_size / sizeof(cell_t) + 1;
   314   jump_map_ = new Label[nmaxops];
   315 }
   316 
   317 Compiler::~Compiler()
   318 {
   319   delete [] jump_map_;
   320 }
   321 
   322 JitFunction *
   323 Compiler::emit(int *errp)
   324 {
   325   if (cip_ >= code_end_ || *cip_ != OP_PROC) {
   326     *errp = SP_ERROR_INVALID_INSTRUCTION;
   327     return NULL;
   328   }
   329 
   330 #if !defined NDEBUG
   331   g_engine1.GetDebugHook()->OnDebugSpew(
   332       "Compiling function %s::%s\n",
   333       plugin_->name,
   334       GetFunctionName(plugin_, pcode_start_));
   335 
   336 #endif
   337 #if defined DEBUG
   338   SpewOpcode(plugin_, code_start_, cip_);
   339 #endif
   340 
   341   cell_t *codeseg = reinterpret_cast<cell_t *>(plugin_->pcode);
   342 
   343   cip_++;
   344   if (!emitOp(OP_PROC)) {
   345       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;
   346       return NULL;
   347   }
   348 
   349   while (cip_ < code_end_) {
   350     // If we reach the end of this function, or the beginning of a new
   351     // procedure, then stop.
   352     if (*cip_ == OP_PROC || *cip_ == OP_ENDPROC)
   353       break;
   354 
   355 #if defined DEBUG
   356     SpewOpcode(plugin_, code_start_, cip_);
   357 #endif
   358 
   359     // We assume every instruction is a jump target, so before emitting
   360     // an opcode, we bind its corresponding label.
   361     __ bind(&jump_map_[cip_ - codeseg]);
   362 
   363     OPCODE op = (OPCODE)readCell();
   364     if (!emitOp(op) || error_ != SP_ERROR_NONE) {
   365       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;
   366       return NULL;
   367     }
   368   }
   369 
   370   emitCallThunks();
   371   emitErrorPaths();
   372 
   373   uint8_t *code = LinkCode(masm);
   374   if (!code) {
   375     *errp = SP_ERROR_OUT_OF_MEMORY;
   376     return NULL;
   377   }
   378 
   379   LoopEdge *edges = new LoopEdge[backward_jumps_.length()];
   380   for (size_t i = 0; i < backward_jumps_.length(); i++) {
   381     edges[i].offset = backward_jumps_[i];
   382     edges[i].disp32 = *reinterpret_cast<int32_t *>(code + edges[i].offset - 4);
   383   }
   384 
   385   return new JitFunction(code, pcode_start_, edges, backward_jumps_.length());
   386 }
   387 
   388 bool
   389 Compiler::emitOp(OPCODE op)
   390 {
   391   switch (op) {
   392     case OP_MOVE_PRI:
   393       __ movl(pri, alt);
   394       break;
   395 
   396     case OP_MOVE_ALT:
   397       __ movl(alt, pri);
   398       break;
   399 
   400     case OP_XCHG:
   401       __ xchgl(pri, alt);
   402       break;
   403 
   404     case OP_ZERO:
   405     {
   406       cell_t offset = readCell();
   407       __ movl(Operand(dat, offset), 0);
   408       break;
   409     }
   410 
   411     case OP_ZERO_S:
   412     {
   413       cell_t offset = readCell();
   414       __ movl(Operand(frm, offset), 0);
   415       break;
   416     }
   417 
   418     case OP_PUSH_PRI:
   419     case OP_PUSH_ALT:
   420     {
   421       Register reg = (op == OP_PUSH_PRI) ? pri : alt;
   422       __ movl(Operand(stk, -4), reg);
   423       __ subl(stk, 4);
   424       break;
   425     }
   426 
   427     case OP_PUSH_C:
   428     case OP_PUSH2_C:
   429     case OP_PUSH3_C:
   430     case OP_PUSH4_C:
   431     case OP_PUSH5_C:
   432     {
   433       int n = 1;
   434       if (op >= OP_PUSH2_C)
   435         n = ((op - OP_PUSH2_C) / 4) + 2;
   436 
   437       int i = 1;
   438       do {
   439         cell_t val = readCell();
   440         __ movl(Operand(stk, -(4 * i)), val);
   441       } while (i++ < n);
   442       __ subl(stk, 4 * n);
   443       break;
   444     }
   445 
   446     case OP_PUSH_ADR:
   447     case OP_PUSH2_ADR:
   448     case OP_PUSH3_ADR:
   449     case OP_PUSH4_ADR:
   450     case OP_PUSH5_ADR:
   451     {
   452       int n = 1;
   453       if (op >= OP_PUSH2_ADR)
   454         n = ((op - OP_PUSH2_ADR) / 4) + 2;
   455 
   456       int i = 1;
   457 
   458       // We temporarily relocate FRM to be a local address instead of an
   459       // absolute address.
   460       __ subl(frm, dat);
   461       do {
   462         cell_t offset = readCell();
   463         __ lea(tmp, Operand(frm, offset));
   464         __ movl(Operand(stk, -(4 * i)), tmp);
   465       } while (i++ < n);
   466       __ subl(stk, 4 * n);
   467       __ addl(frm, dat);
   468       break;
   469     }
   470 
   471     case OP_PUSH_S:
   472     case OP_PUSH2_S:
   473     case OP_PUSH3_S:
   474     case OP_PUSH4_S:
   475     case OP_PUSH5_S:
   476     {
   477       int n = 1;
   478       if (op >= OP_PUSH2_S)
   479         n = ((op - OP_PUSH2_S) / 4) + 2;
   480 
   481       int i = 1;
   482       do {
   483         cell_t offset = readCell();
   484         __ movl(tmp, Operand(frm, offset));
   485         __ movl(Operand(stk, -(4 * i)), tmp);
   486       } while (i++ < n);
   487       __ subl(stk, 4 * n);
   488       break;
   489     }
   490 
   491     case OP_PUSH:
   492     case OP_PUSH2:
   493     case OP_PUSH3:
   494     case OP_PUSH4:
   495     case OP_PUSH5:
   496     {
   497       int n = 1;
   498       if (op >= OP_PUSH2)
   499         n = ((op - OP_PUSH2) / 4) + 2;
   500 
   501       int i = 1;
   502       do {
   503         cell_t offset = readCell();
   504         __ movl(tmp, Operand(dat, offset));
   505         __ movl(Operand(stk, -(4 * i)), tmp);
   506       } while (i++ < n);
   507       __ subl(stk, 4 * n);
   508       break;
   509     }
   510 
   511     case OP_ZERO_PRI:
   512       __ xorl(pri, pri);
   513       break;
   514 
   515     case OP_ZERO_ALT:
   516       __ xorl(alt, alt);
   517       break;
   518 
   519     case OP_ADD:
   520       __ addl(pri, alt);
   521       break;
   522 
   523     case OP_SUB:
   524       __ subl(pri, alt);
   525       break;
   526 
   527     case OP_SUB_ALT:
   528       __ movl(tmp, alt);
   529       __ subl(tmp, pri);
   530       __ movl(pri, tmp);
   531       break;
   532 
   533     case OP_PROC:
   534       // Push the old frame onto the stack.
   535       __ movl(tmp, Operand(frmAddr()));
   536       __ movl(Operand(stk, -4), tmp);
   537       __ subl(stk, 8);    // extra unused slot for non-existant CIP
   538 
   539       // Get and store the new frame.
   540       __ movl(tmp, stk);
   541       __ movl(frm, stk);
   542       __ subl(tmp, dat);
   543       __ movl(Operand(frmAddr()), tmp);
   544 
   545       // Align the stack to 16-bytes (each call adds 4 bytes).
   546       __ subl(esp, 12);
   547       break;
   548 
   549     case OP_IDXADDR_B:
   550     {
   551       cell_t val = readCell();
   552       __ shll(pri, val);
   553       __ addl(pri, alt);
   554       break;
   555     }
   556 
   557     case OP_SHL:
   558       __ movl(ecx, alt);
   559       __ shll_cl(pri);
   560       break;
   561 
   562     case OP_SHR:
   563       __ movl(ecx, alt);
   564       __ shrl_cl(pri);
   565       break;
   566 
   567     case OP_SSHR:
   568       __ movl(ecx, alt);
   569       __ sarl_cl(pri);
   570       break;
   571 
   572     case OP_SHL_C_PRI:
   573     case OP_SHL_C_ALT:
   574     {
   575       Register reg = (op == OP_SHL_C_PRI) ? pri : alt;
   576       cell_t val = readCell();
   577       __ shll(reg, val);
   578       break;
   579     }
   580 
   581     case OP_SHR_C_PRI:
   582     case OP_SHR_C_ALT:
   583     {
   584       Register reg = (op == OP_SHR_C_PRI) ? pri : alt;
   585       cell_t val = readCell();
   586       __ shrl(reg, val);
   587       break;
   588     }
   589 
   590     case OP_SMUL:
   591       __ imull(pri, alt);
   592       break;
   593 
   594     case OP_NOT:
   595       __ testl(eax, eax);
   596       __ movl(eax, 0);
   597       __ set(zero, r8_al);
   598       break;
   599 
   600     case OP_NEG:
   601       __ negl(eax);
   602       break;
   603 
   604     case OP_XOR:
   605       __ xorl(pri, alt);
   606       break;
   607 
   608     case OP_OR:
   609       __ orl(pri, alt);
   610       break;
   611 
   612     case OP_AND:
   613       __ andl(pri, alt);
   614       break;
   615 
   616     case OP_INVERT:
   617       __ notl(pri);
   618       break;
   619 
   620     case OP_ADD_C:
   621     {
   622       cell_t val = readCell();
   623       __ addl(pri, val);
   624       break;
   625     }
   626 
   627     case OP_SMUL_C:
   628     {
   629       cell_t val = readCell();
   630       __ imull(pri, pri, val);
   631       break;
   632     }
   633 
   634     case OP_EQ:
   635     case OP_NEQ:
   636     case OP_SLESS:
   637     case OP_SLEQ:
   638     case OP_SGRTR:
   639     case OP_SGEQ:
   640     {
   641       ConditionCode cc = OpToCondition(op);
   642       __ cmpl(pri, alt);
   643       __ movl(pri, 0);
   644       __ set(cc, r8_al);
   645       break;
   646     }
   647 
   648     case OP_EQ_C_PRI:
   649     case OP_EQ_C_ALT:
   650     {
   651       Register reg = (op == OP_EQ_C_PRI) ? pri : alt;
   652       cell_t val = readCell();
   653       __ cmpl(reg, val);
   654       __ movl(pri, 0);
   655       __ set(equal, r8_al);
   656       break;
   657     }
   658 
   659     case OP_INC_PRI:
   660     case OP_INC_ALT:
   661     {
   662       Register reg = (OP_INC_PRI) ? pri : alt;
   663       __ addl(reg, 1);
   664       break;
   665     }
   666 
   667     case OP_INC:
   668     case OP_INC_S:
   669     {
   670       Register base = (op == OP_INC) ? dat : frm;
   671       cell_t offset = readCell();
   672       __ addl(Operand(base, offset), 1);
   673       break;
   674     }
   675 
   676     case OP_INC_I:
   677       __ addl(Operand(dat, pri, NoScale), 1);
   678       break;
   679 
   680     case OP_DEC_PRI:
   681     case OP_DEC_ALT:
   682     {
   683       Register reg = (op == OP_DEC_PRI) ? pri : alt;
   684       __ subl(reg, 1);
   685       break;
   686     }
   687 
   688     case OP_DEC:
   689     case OP_DEC_S:
   690     {
   691       Register base = (op == OP_DEC) ? dat : frm;
   692       cell_t offset = readCell();
   693       __ subl(Operand(base, offset), 1);
   694       break;
   695     }
   696 
   697     case OP_DEC_I:
   698       __ subl(Operand(dat, pri, NoScale), 1);
   699       break;
   700 
   701     case OP_LOAD_PRI:
   702     case OP_LOAD_ALT:
   703     {
   704       Register reg = (op == OP_LOAD_PRI) ? pri : alt;
   705       cell_t offset = readCell();
   706       __ movl(reg, Operand(dat, offset));
   707       break;
   708     }
   709 
   710     case OP_LOAD_BOTH:
   711     {
   712       cell_t offs1 = readCell();
   713       cell_t offs2 = readCell();
   714       __ movl(pri, Operand(dat, offs1));
   715       __ movl(alt, Operand(dat, offs2));
   716       break;
   717     }
   718 
   719     case OP_LOAD_S_PRI:
   720     case OP_LOAD_S_ALT:
   721     {
   722       Register reg = (op == OP_LOAD_S_PRI) ? pri : alt;
   723       cell_t offset = readCell();
   724       __ movl(reg, Operand(frm, offset));
   725       break;
   726     }
   727 
   728     case OP_LOAD_S_BOTH:
   729     {
   730       cell_t offs1 = readCell();
   731       cell_t offs2 = readCell();
   732       __ movl(pri, Operand(frm, offs1));
   733       __ movl(alt, Operand(frm, offs2));
   734       break;
   735     }
   736 
   737     case OP_LREF_S_PRI:
   738     case OP_LREF_S_ALT:
   739     {
   740       Register reg = (op == OP_LREF_S_PRI) ? pri : alt;
   741       cell_t offset = readCell();
   742       __ movl(reg, Operand(frm, offset));
   743       __ movl(reg, Operand(dat, reg, NoScale));
   744       break;
   745     }
   746 
   747     case OP_CONST_PRI:
   748     case OP_CONST_ALT:
   749     {
   750       Register reg = (op == OP_CONST_PRI) ? pri : alt;
   751       cell_t val = readCell();
   752       __ movl(reg, val);
   753       break;
   754     }
   755 
   756     case OP_ADDR_PRI:
   757     case OP_ADDR_ALT:
   758     {
   759       Register reg = (op == OP_ADDR_PRI) ? pri : alt;
   760       cell_t offset = readCell();
   761       __ movl(reg, Operand(frmAddr()));
   762       __ addl(reg, offset);
   763       break;
   764     }
   765 
   766     case OP_STOR_PRI:
   767     case OP_STOR_ALT:
   768     {
   769       Register reg = (op == OP_STOR_PRI) ? pri : alt;
   770       cell_t offset = readCell();
   771       __ movl(Operand(dat, offset), reg);
   772       break;
   773     }
   774 
   775     case OP_STOR_S_PRI:
   776     case OP_STOR_S_ALT:
   777     {
   778       Register reg = (op == OP_STOR_S_PRI) ? pri : alt;
   779       cell_t offset = readCell();
   780       __ movl(Operand(frm, offset), reg);
   781       break;
   782     }
   783 
   784     case OP_IDXADDR:
   785       __ lea(pri, Operand(alt, pri, ScaleFour));
   786       break;
   787 
   788     case OP_SREF_S_PRI:
   789     case OP_SREF_S_ALT:
   790     {
   791       Register reg = (op == OP_SREF_S_PRI) ? pri : alt;
   792       cell_t offset = readCell();
   793       __ movl(tmp, Operand(frm, offset));
   794       __ movl(Operand(dat, tmp, NoScale), reg);
   795       break;
   796     }
   797 
   798     case OP_POP_PRI:
   799     case OP_POP_ALT:
   800     {
   801       Register reg = (op == OP_POP_PRI) ? pri : alt;
   802       __ movl(reg, Operand(stk, 0));
   803       __ addl(stk, 4);
   804       break;
   805     }
   806 
   807     case OP_SWAP_PRI:
   808     case OP_SWAP_ALT:
   809     {
   810       Register reg = (op == OP_SWAP_PRI) ? pri : alt;
   811       __ movl(tmp, Operand(stk, 0));
   812       __ movl(Operand(stk, 0), reg);
   813       __ movl(reg, tmp);
   814       break;
   815     }
   816 
   817     case OP_LIDX:
   818       __ lea(pri, Operand(alt, pri, ScaleFour));
   819       __ movl(pri, Operand(dat, pri, NoScale));
   820       break;
   821 
   822     case OP_LIDX_B:
   823     {
   824       cell_t val = readCell();
   825       if (val >= 0 && val <= 3) {
   826         __ lea(pri, Operand(alt, pri, Scale(val)));
   827       } else {
   828         __ shll(pri, val);
   829         __ addl(pri, alt);
   830       }
   831       emitCheckAddress(pri);
   832       __ movl(pri, Operand(dat, pri, NoScale));
   833       break;
   834     }
   835 
   836     case OP_CONST:
   837     case OP_CONST_S:
   838     {
   839       Register base = (op == OP_CONST) ? dat : frm;
   840       cell_t offset = readCell();
   841       cell_t val = readCell();
   842       __ movl(Operand(base, offset), val);
   843       break;
   844     }
   845 
   846     case OP_LOAD_I:
   847       emitCheckAddress(pri);
   848       __ movl(pri, Operand(dat, pri, NoScale));
   849       break;
   850 
   851     case OP_STOR_I:
   852       emitCheckAddress(alt);
   853       __ movl(Operand(dat, alt, NoScale), pri);
   854       break;
   855 
   856     case OP_SDIV:
   857     case OP_SDIV_ALT:
   858     {
   859       Register dividend = (op == OP_SDIV) ? pri : alt;
   860       Register divisor = (op == OP_SDIV) ? alt : pri;
   861 
   862       // Guard against divide-by-zero.
   863       __ testl(divisor, divisor);
   864       __ j(zero, &error_divide_by_zero_);
   865 
   866       // A more subtle case; -INT_MIN / -1 yields an overflow exception.
   867       Label ok;
   868       __ cmpl(divisor, -1);
   869       __ j(not_equal, &ok);
   870       __ cmpl(dividend, 0x80000000);
   871       __ j(equal, &error_integer_overflow_);
   872       __ bind(&ok);
   873 
   874       // Now we can actually perform the divide.
   875       __ movl(tmp, divisor);
   876       if (op == OP_SDIV)
   877         __ movl(edx, dividend);
   878       else
   879         __ movl(eax, dividend);
   880       __ sarl(edx, 31);
   881       __ idivl(tmp);
   882       break;
   883     }
   884 
   885     case OP_LODB_I:
   886     {
   887       cell_t val = readCell();
   888       emitCheckAddress(pri);
   889       __ movl(pri, Operand(dat, pri, NoScale));
   890       if (val == 1)
   891         __ andl(pri, 0xff);
   892       else if (val == 2)
   893         __ andl(pri, 0xffff);
   894       break;
   895     }
   896 
   897     case OP_STRB_I:
   898     {
   899       cell_t val = readCell();
   900       emitCheckAddress(alt);
   901       if (val == 1)
   902         __ movb(Operand(dat, alt, NoScale), pri);
   903       else if (val == 2)
   904         __ movw(Operand(dat, alt, NoScale), pri);
   905       else if (val == 4)
   906         __ movl(Operand(dat, alt, NoScale), pri);
   907       break;
   908     }
   909 
   910     case OP_RETN:
   911     {
   912       // Restore the old frame pointer.
   913       __ movl(frm, Operand(stk, 4));              // get the old frm
   914       __ addl(stk, 8);                            // pop stack
   915       __ movl(Operand(frmAddr()), frm);           // store back old frm
   916       __ addl(frm, dat);                          // relocate
   917 
   918       // Remove parameters.
   919       __ movl(tmp, Operand(stk, 0));
   920       __ lea(stk, Operand(stk, tmp, ScaleFour, 4));
   921       __ addl(esp, 12);
   922       __ ret();
   923       break;
   924     }
   925 
   926     case OP_MOVS:
   927     {
   928       cell_t val = readCell();
   929       unsigned dwords = val / 4;
   930       unsigned bytes = val % 4;
   931 
   932       __ cld();
   933       __ push(esi);
   934       __ push(edi);
   935       // Note: set edi first, since we need esi.
   936       __ lea(edi, Operand(dat, alt, NoScale));
   937       __ lea(esi, Operand(dat, pri, NoScale));
   938       if (dwords) {
   939         __ movl(ecx, dwords);
   940         __ rep_movsd();
   941       }
   942       if (bytes) {
   943         __ movl(ecx, bytes);
   944         __ rep_movsb();
   945       }
   946       __ pop(edi);
   947       __ pop(esi);
   948       break;
   949     }
   950 
   951     case OP_FILL:
   952     {
   953       // eax/pri is used implicitly.
   954       unsigned dwords = readCell() / 4;
   955       __ push(edi);
   956       __ lea(edi, Operand(dat, alt, NoScale));
   957       __ movl(ecx, dwords);
   958       __ cld();
   959       __ rep_stosd();
   960       __ pop(edi);
   961       break;
   962     }
   963 
   964     case OP_STRADJUST_PRI:
   965       __ addl(pri, 4);
   966       __ sarl(pri, 2);
   967       break;
   968 
   969     case OP_FABS:
   970       __ movl(pri, Operand(stk, 0));
   971       __ andl(pri, 0x7fffffff);
   972       __ addl(stk, 4);
   973       break;
   974 
   975     case OP_FLOAT:
   976       if (MacroAssemblerX86::Features().sse2) {
   977         __ cvtsi2ss(xmm0, Operand(edi, 0));
   978         __ movd(pri, xmm0);
   979       } else {
   980         __ fild32(Operand(edi, 0));
   981         __ subl(esp, 4);
   982         __ fstp32(Operand(esp, 0));
   983         __ pop(pri);
   984       }
   985       __ addl(stk, 4);
   986       break;
   987 
   988     case OP_FLOATADD:
   989     case OP_FLOATSUB:
   990     case OP_FLOATMUL:
   991     case OP_FLOATDIV:
   992       if (MacroAssemblerX86::Features().sse2) {
   993         __ movss(xmm0, Operand(stk, 0));
   994         if (op == OP_FLOATADD)
   995           __ addss(xmm0, Operand(stk, 4));
   996         else if (op == OP_FLOATSUB)
   997           __ subss(xmm0, Operand(stk, 4));
   998         else if (op == OP_FLOATMUL)
   999           __ mulss(xmm0, Operand(stk, 4));
  1000         else if (op == OP_FLOATDIV)
  1001           __ divss(xmm0, Operand(stk, 4));
  1002         __ movd(pri, xmm0);
  1003       } else {
  1004         __ subl(esp, 4);
  1005         __ fld32(Operand(stk, 0));
  1006 
  1007         if (op == OP_FLOATADD)
  1008           __ fadd32(Operand(stk, 4));
  1009         else if (op == OP_FLOATSUB)
  1010           __ fsub32(Operand(stk, 4));
  1011         else if (op == OP_FLOATMUL)
  1012           __ fmul32(Operand(stk, 4));
  1013         else if (op == OP_FLOATDIV)
  1014           __ fdiv32(Operand(stk, 4));
  1015 
  1016         __ fstp32(Operand(esp, 0));
  1017         __ pop(pri);
  1018       }
  1019       __ addl(stk, 8);
  1020       break;
  1021 
  1022     case OP_RND_TO_NEAREST:
  1023     {
  1024       if (MacroAssemblerX86::Features().sse) {
  1025         // Assume no one is touching MXCSR.
  1026         __ cvtss2si(pri, Operand(stk, 0));
  1027       } else {
  1028         static float kRoundToNearest = 0.5f;
  1029         // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf
  1030         __ fld32(Operand(stk, 0));
  1031         __ fadd32(st0, st0);
  1032         __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));
  1033         __ subl(esp, 4);
  1034         __ fistp32(Operand(esp, 0));
  1035         __ pop(pri);
  1036         __ sarl(pri, 1);
  1037       }
  1038       __ addl(stk, 4);
  1039       break;
  1040     }
  1041 
  1042     case OP_RND_TO_CEIL:
  1043     {
  1044       static float kRoundToCeil = -0.5f;
  1045       // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf
  1046       __ fld32(Operand(stk, 0));
  1047       __ fadd32(st0, st0);
  1048       __ fsubr32(Operand(ExternalAddress(&kRoundToCeil)));
  1049       __ subl(esp, 4);
  1050       __ fistp32(Operand(esp, 0));
  1051       __ pop(pri);
  1052       __ sarl(pri, 1);
  1053       __ negl(pri);
  1054       __ addl(stk, 4);
  1055       break;
  1056     }
  1057 
  1058     case OP_RND_TO_ZERO:
  1059       if (MacroAssemblerX86::Features().sse) {
  1060         __ cvttss2si(pri, Operand(stk, 0));
  1061       } else {
  1062         __ fld32(Operand(stk, 0));
  1063         __ subl(esp, 8);
  1064         __ fstcw(Operand(esp, 4));
  1065         __ movl(Operand(esp, 0), 0xfff);
  1066         __ fldcw(Operand(esp, 0));
  1067         __ fistp32(Operand(esp, 0));
  1068         __ pop(pri);
  1069         __ fldcw(Operand(esp, 0));
  1070         __ addl(esp, 4);
  1071       }
  1072       __ addl(stk, 4);
  1073       break;
  1074 
  1075     case OP_RND_TO_FLOOR:
  1076       __ fld32(Operand(stk, 0));
  1077       __ subl(esp, 8);
  1078       __ fstcw(Operand(esp, 4));
  1079       __ movl(Operand(esp, 0), 0x7ff);
  1080       __ fldcw(Operand(esp, 0));
  1081       __ fistp32(Operand(esp, 0));
  1082       __ pop(eax);
  1083       __ fldcw(Operand(esp, 0));
  1084       __ addl(esp, 4);
  1085       __ addl(stk, 4);
  1086       break;
  1087 
  1088     case OP_FLOATCMP:
  1089     {
  1090       Label bl, ab, done;
  1091       if (MacroAssemblerX86::Features().sse) {
  1092         __ movss(xmm0, Operand(stk, 4));
  1093         __ ucomiss(xmm0, Operand(stk, 0));
  1094       } else {
  1095         __ fld32(Operand(stk, 0));
  1096         __ fld32(Operand(stk, 4));
  1097         __ fucomip(st1);
  1098         __ fstp(st0);
  1099       }
  1100       __ j(above, &ab);
  1101       __ j(below, &bl);
  1102       __ xorl(pri, pri);
  1103       __ jmp(&done);
  1104       __ bind(&ab);
  1105       __ movl(pri, -1);
  1106       __ jmp(&done);
  1107       __ bind(&bl);
  1108       __ movl(pri, 1);
  1109       __ bind(&done);
  1110       __ addl(stk, 8);
  1111       break;
  1112     }
  1113 
  1114     case OP_STACK:
  1115     {
  1116       cell_t amount = readCell();
  1117       __ addl(stk, amount);
  1118 
  1119      if (amount > 0) {
  1120        // Check if the stack went beyond the stack top - usually a compiler error.
  1121        __ cmpl(stk, intptr_t(plugin_->memory + plugin_->mem_size));
  1122        __ j(not_below, &error_stack_min_);
  1123      } else {
  1124        // Check if the stack is going to collide with the heap.
  1125        __ movl(tmp, Operand(hpAddr()));
  1126        __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));
  1127        __ cmpl(stk, tmp);
  1128        __ j(below, &error_stack_low_);
  1129      }
  1130      break;
  1131     }
  1132 
  1133     case OP_HEAP:
  1134     {
  1135       cell_t amount = readCell();
  1136       __ movl(alt, Operand(hpAddr()));
  1137       __ addl(Operand(hpAddr()), amount);
  1138 
  1139       if (amount < 0) {
  1140         __ cmpl(Operand(hpAddr()), plugin_->data_size);
  1141         __ j(below, &error_heap_min_);
  1142       } else {
  1143         __ movl(tmp, Operand(hpAddr()));
  1144         __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));
  1145         __ cmpl(tmp, stk);
  1146         __ j(above, &error_heap_low_);
  1147       }
  1148       break;
  1149     }
  1150 
  1151     case OP_JUMP:
  1152     {
  1153       Label *target = labelAt(readCell());
  1154       if (!target)
  1155         return false;
  1156       if (target->bound()) {
  1157         __ jmp32(target);
  1158         backward_jumps_.append(masm.pc());
  1159       } else {
  1160         __ jmp(target);
  1161       }
  1162       break;
  1163     }
  1164 
  1165     case OP_JZER:
  1166     case OP_JNZ:
  1167     {
  1168       ConditionCode cc = (op == OP_JZER) ? zero : not_zero;
  1169       Label *target = labelAt(readCell());
  1170       if (!target)
  1171         return false;
  1172       __ testl(pri, pri);
  1173       if (target->bound()) {
  1174         __ j32(cc, target);
  1175         backward_jumps_.append(masm.pc());
  1176       } else {
  1177         __ j(cc, target);
  1178       }
  1179       break;
  1180     }
  1181 
  1182     case OP_JEQ:
  1183     case OP_JNEQ:
  1184     case OP_JSLESS:
  1185     case OP_JSLEQ:
  1186     case OP_JSGRTR:
  1187     case OP_JSGEQ:
  1188     {
  1189       Label *target = labelAt(readCell());
  1190       if (!target)
  1191         return false;
  1192       ConditionCode cc = OpToCondition(op);
  1193       __ cmpl(pri, alt);
  1194       if (target->bound()) {
  1195         __ j32(cc, target);
  1196         backward_jumps_.append(masm.pc());
  1197       } else {
  1198         __ j(cc, target);
  1199       }
  1200       break;
  1201     }
  1202 
  1203     case OP_TRACKER_PUSH_C:
  1204     {
  1205       cell_t amount = readCell();
  1206 
  1207       __ push(pri);
  1208       __ push(alt);
  1209 
  1210       __ push(amount * 4);
  1211       __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));
  1212       __ call(ExternalAddress((void *)PushTracker));
  1213       __ addl(esp, 8);
  1214       __ testl(eax, eax);
  1215       __ j(not_zero, &extern_error_);
  1216 
  1217       __ pop(alt);
  1218       __ pop(pri);
  1219       break;
  1220     }
  1221 
  1222     case OP_TRACKER_POP_SETHEAP:
  1223     {
  1224       // Save registers.
  1225       __ push(pri);
  1226       __ push(alt);
  1227 
  1228       // Get the context pointer and call the sanity checker.
  1229       __ push(intptr_t(rt_));
  1230       __ call(ExternalAddress((void *)PopTrackerAndSetHeap));
  1231       __ addl(esp, 4);
  1232       __ testl(eax, eax);
  1233       __ j(not_zero, &extern_error_);
  1234 
  1235       __ pop(alt);
  1236       __ pop(pri);
  1237       break;
  1238     }
  1239 
  1240     case OP_BREAK:
  1241     {
  1242       cell_t cip = uintptr_t(cip_ - 1) - uintptr_t(plugin_->pcode);
  1243       __ movl(Operand(cipAddr()), cip);
  1244       break;
  1245     }
  1246 
  1247     case OP_HALT:
  1248       __ align(16);
  1249       __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1250       __ movl(Operand(tmp, offsetof(sp_context_t, rval)), pri);
  1251       __ movl(pri, readCell());
  1252       __ jmp(&extern_error_);
  1253       break;
  1254 
  1255     case OP_BOUNDS:
  1256     {
  1257       cell_t value = readCell();
  1258       __ cmpl(eax, value);
  1259       __ j(above, &error_bounds_);
  1260       break;
  1261     }
  1262 
  1263     case OP_GENARRAY:
  1264     case OP_GENARRAY_Z:
  1265       emitGenArray(op == OP_GENARRAY_Z);
  1266       break;
  1267 
  1268     case OP_CALL:
  1269       if (!emitCall())
  1270         return false;
  1271       break;
  1272 
  1273     case OP_SYSREQ_C:
  1274     case OP_SYSREQ_N:
  1275       if (!emitNativeCall(op))
  1276         return false;
  1277       break;
  1278 
  1279     case OP_SWITCH:
  1280       if (!emitSwitch())
  1281         return false;
  1282       break;
  1283 
  1284     case OP_CASETBL:
  1285     {
  1286       size_t ncases = readCell();
  1287 
  1288       // Two cells per case, and one extra cell for the default address.
  1289       cip_ += (ncases * 2) + 1;
  1290       break;
  1291     }
  1292 
  1293     case OP_NOP:
  1294       break;
  1295 
  1296     default:
  1297       error_ = SP_ERROR_INVALID_INSTRUCTION;
  1298       return false;
  1299   }
  1300 
  1301   return true;
  1302 }
  1303 
  1304 Label *
  1305 Compiler::labelAt(size_t offset)
  1306 {
  1307   if (offset % 4 != 0 ||
  1308       offset > plugin_->pcode_size ||
  1309       offset <= pcode_start_)
  1310   {
  1311     // If the jump target is misaligned, or out of pcode bounds, or is an
  1312     // address out of the function bounds, we abort. Unfortunately we can't
  1313     // test beyond the end of the function since we don't have a precursor
  1314     // pass (yet).
  1315     error_ = SP_ERROR_INSTRUCTION_PARAM;
  1316     return NULL;
  1317   }
  1318 
  1319   return &jump_map_[offset / sizeof(cell_t)];
  1320 }
  1321 
  1322 void
  1323 Compiler::emitCheckAddress(Register reg)
  1324 {
  1325   // Check if we're in memory bounds.
  1326   __ cmpl(reg, plugin_->mem_size);
  1327   __ j(not_below, &error_memaccess_);
  1328 
  1329   // Check if we're in the invalid region between hp and sp.
  1330   Label done;
  1331   __ cmpl(reg, Operand(hpAddr()));
  1332   __ j(below, &done);
  1333   __ lea(tmp, Operand(dat, reg, NoScale));
  1334   __ cmpl(tmp, stk);
  1335   __ j(below, &error_memaccess_);
  1336   __ bind(&done);
  1337 }
  1338 
  1339 void
  1340 Compiler::emitGenArray(bool autozero)
  1341 {
  1342   cell_t val = readCell();
  1343   if (val == 1)
  1344   {
  1345     // flat array; we can generate this without indirection tables.
  1346     // Note that we can overwrite ALT because technically STACK should be destroying ALT
  1347     __ movl(alt, Operand(hpAddr()));
  1348     __ movl(tmp, Operand(stk, 0));
  1349     __ movl(Operand(stk, 0), alt);    // store base of the array into the stack.
  1350     __ lea(alt, Operand(alt, tmp, ScaleFour));
  1351     __ movl(Operand(hpAddr()), alt);
  1352     __ addl(alt, dat);
  1353     __ cmpl(alt, stk);
  1354     __ j(not_below, &error_heap_low_);
  1355 
  1356     __ shll(tmp, 2);
  1357     __ push(tmp);
  1358     __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));
  1359     __ call(ExternalAddress((void *)PushTracker));
  1360     __ addl(esp, 4);
  1361     __ pop(tmp);
  1362     __ shrl(tmp, 2);
  1363     __ testl(eax, eax);
  1364     __ j(not_zero, &extern_error_);
  1365 
  1366     if (autozero) {
  1367       // Note - tmp is ecx and still intact.
  1368       __ push(eax);
  1369       __ push(edi);
  1370       __ xorl(eax, eax);
  1371       __ movl(edi, Operand(stk, 0));
  1372       __ addl(edi, dat);
  1373       __ cld();
  1374       __ rep_stosd();
  1375       __ pop(edi);
  1376       __ pop(eax);
  1377     }
  1378   } else {
  1379     __ push(pri);
  1380 
  1381     // int GenerateArray(sp_plugin_t, vars[], uint32_t, cell_t *, int, unsigned *);
  1382     __ push(autozero ? 1 : 0);
  1383     __ push(stk);
  1384     __ push(val);
  1385     __ push(intptr_t(rt_));
  1386     __ call(ExternalAddress((void *)GenerateFullArray));
  1387     __ addl(esp, 4 * sizeof(void *));
  1388 
  1389     // restore pri to tmp
  1390     __ pop(tmp);
  1391 
  1392     __ testl(eax, eax);
  1393     __ j(not_zero, &extern_error_);
  1394 
  1395     // Move tmp back to pri, remove pushed args.
  1396     __ movl(pri, tmp);
  1397     __ addl(stk, (val - 1) * 4);
  1398   }
  1399 }
  1400 
  1401 bool
  1402 Compiler::emitCall()
  1403 {
  1404   cell_t offset = readCell();
  1405 
  1406   // If this offset looks crappy, i.e. not aligned or out of bounds, we just
  1407   // abort.
  1408   if (offset % 4 != 0 || uint32_t(offset) >= plugin_->pcode_size) {
  1409     error_ = SP_ERROR_INSTRUCTION_PARAM;
  1410     return false;
  1411   }
  1412 
  1413   // eax = context
  1414   // ecx = rp
  1415   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1416   __ movl(ecx, Operand(eax, offsetof(sp_context_t, rp)));
  1417 
  1418   // Check if the return stack is used up.
  1419   __ cmpl(ecx, SP_MAX_RETURN_STACK);
  1420   __ j(not_below, &error_stack_low_);
  1421 
  1422   // Add to the return stack.
  1423   uintptr_t cip = uintptr_t(cip_ - 2) - uintptr_t(plugin_->pcode);
  1424   __ movl(Operand(eax, ecx, ScaleFour, offsetof(sp_context_t, rstk_cips)), cip);
  1425 
  1426   // Increment the return stack pointer.
  1427   __ addl(Operand(eax, offsetof(sp_context_t, rp)), 1);
  1428 
  1429   // Store the CIP of the function we're about to call.
  1430   __ movl(Operand(cipAddr()), offset);
  1431 
  1432   JitFunction *fun = rt_->GetJittedFunctionByOffset(offset);
  1433   if (!fun) {
  1434     // Need to emit a delayed thunk.
  1435     CallThunk *thunk = new CallThunk(offset);
  1436     __ call(&thunk->call);
  1437     if (!thunks_.append(thunk))
  1438       return false;
  1439   } else {
  1440     // Function is already emitted, we can do a direct call.
  1441     __ call(ExternalAddress(fun->GetEntryAddress()));
  1442   }
  1443 
  1444   // Restore the last cip.
  1445   __ movl(Operand(cipAddr()), cip);
  1446 
  1447   // Mark us as leaving the last frame.
  1448   __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1449   __ subl(Operand(tmp, offsetof(sp_context_t, rp)), 1);
  1450   return true;
  1451 }
  1452 
  1453 void
  1454 Compiler::emitCallThunks()
  1455 {
  1456   for (size_t i = 0; i < thunks_.length(); i++) {
  1457     CallThunk *thunk = thunks_[i];
  1458 
  1459     Label error;
  1460     __ bind(&thunk->call);
  1461     // Huge hack - get the return address, since that is the call that we
  1462     // need to patch.
  1463     __ movl(eax, Operand(esp, 0));
  1464 
  1465     // We need to push 4 arguments, and one of them will need an extra word
  1466     // on the stack. Allocate a big block so we're aligned, subtracting
  1467     // 4 because we got here via a call.
  1468     static const size_t kStackNeeded = 5 * sizeof(void *);
  1469     static const size_t kStackReserve = ke::Align(kStackNeeded, 16) - sizeof(void *);
  1470     __ subl(esp, kStackReserve);
  1471 
  1472     // Set arguments.
  1473     __ movl(Operand(esp, 3 * sizeof(void *)), eax);
  1474     __ lea(edx, Operand(esp, 4 * sizeof(void *)));
  1475     __ movl(Operand(esp, 2 * sizeof(void *)), edx);
  1476     __ movl(Operand(esp, 1 * sizeof(void *)), intptr_t(thunk->pcode_offset));
  1477     __ movl(Operand(esp, 0 * sizeof(void *)), intptr_t(rt_));
  1478 
  1479     __ call(ExternalAddress((void *)CompileFromThunk));
  1480     __ movl(edx, Operand(esp, 4 * sizeof(void *)));
  1481     __ addl(esp, kStackReserve);
  1482     __ testl(eax, eax);
  1483     __ j(not_zero, &error);
  1484     __ jmp(edx);
  1485 
  1486     __ bind(&error);
  1487     __ movl(Operand(cipAddr()), thunk->pcode_offset);
  1488     __ jmp(g_Jit.GetUniversalReturn());
  1489   }
  1490 }
  1491 
  1492 cell_t
  1493 Compiler::readCell()
  1494 {
  1495   if (cip_ >= code_end_) {
  1496     error_= SP_ERROR_INVALID_INSTRUCTION;
  1497     return 0;
  1498   }
  1499   return *cip_++;
  1500 }
  1501 
  1502 bool
  1503 Compiler::emitNativeCall(OPCODE op)
  1504 {
  1505   uint32_t native_index = readCell();
  1506 
  1507   if (native_index >= plugin_->num_natives) {
  1508     error_ = SP_ERROR_INSTRUCTION_PARAM;
  1509     return false;
  1510   }
  1511 
  1512   uint32_t num_params;
  1513   if (op == OP_SYSREQ_N) {
  1514     num_params = readCell();
  1515 
  1516     // See if we can get a replacement opcode. If we can, then recursively
  1517     // call emitOp() to generate it. Note: it's important that we do this
  1518     // before generating any code for the SYSREQ.N.
  1519     unsigned replacement = rt_->GetNativeReplacement(native_index);
  1520     if (replacement != OP_NOP)
  1521       return emitOp((OPCODE)replacement);
  1522 
  1523     // Store the number of parameters.
  1524     __ movl(Operand(stk, -4), num_params);
  1525     __ subl(stk, 4);
  1526   }
  1527 
  1528   // Save registers.
  1529   __ push(edx);
  1530 
  1531   // Push the parameters for the C++ function.
  1532   __ push(stk);
  1533   __ push(native_index);
  1534 
  1535   // Relocate our absolute stk to be dat-relative, and update the context's
  1536   // view.
  1537   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1538   __ subl(stk, dat);
  1539   __ movl(Operand(eax, offsetof(sp_context_t, sp)), stk);
  1540 
  1541   // Push context and make the call.
  1542   __ push(eax);
  1543   __ call(ExternalAddress((void *)NativeCallback));
  1544 
  1545   // Check for errors.
  1546   __ movl(ecx, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1547   __ movl(ecx, Operand(ecx, offsetof(sp_context_t, n_err)));
  1548   __ testl(ecx, ecx);
  1549   __ j(not_zero, &extern_error_);
  1550   
  1551   // Restore local state.
  1552   __ addl(stk, dat);
  1553   __ addl(esp, 12);
  1554   __ pop(edx);
  1555 
  1556   if (op == OP_SYSREQ_N) {
  1557     // Pop the stack. Do not check the margins.
  1558     __ addl(stk, (num_params + 1) * sizeof(cell_t));
  1559   }
  1560   return true;
  1561 }
  1562 
  1563 bool
  1564 Compiler::emitSwitch()
  1565 {
  1566   cell_t offset = readCell();
  1567   if (!labelAt(offset))
  1568     return false;
  1569 
  1570   cell_t *tbl = (cell_t *)((char *)plugin_->pcode + offset + sizeof(cell_t));
  1571 
  1572   struct Entry {
  1573     cell_t val;
  1574     cell_t offset;
  1575   };
  1576 
  1577   size_t ncases = *tbl++;
  1578 
  1579   Label *defaultCase = labelAt(*tbl);
  1580   if (!defaultCase)
  1581     return false;
  1582 
  1583   // Degenerate - 0 cases.
  1584   if (!ncases) {
  1585     __ jmp(defaultCase);
  1586     return true;
  1587   }
  1588 
  1589   Entry *cases = (Entry *)(tbl + 1);
  1590 
  1591   // Degenerate - 1 case.
  1592   if (ncases == 1) {
  1593     Label *maybe = labelAt(cases[0].offset);
  1594     if (!maybe)
  1595       return false;
  1596     __ cmpl(pri, cases[0].val);
  1597     __ j(equal, maybe);
  1598     __ jmp(defaultCase);
  1599     return true;
  1600   }
  1601 
  1602   // We have two or more cases, so let's generate a full switch. Decide
  1603   // whether we'll make an if chain, or a jump table, based on whether
  1604   // the numbers are strictly sequential.
  1605   bool sequential = true;
  1606   {
  1607     cell_t first = cases[0].val;
  1608     cell_t last = first;
  1609     for (size_t i = 1; i < ncases; i++) {
  1610       if (cases[i].val != ++last) {
  1611         sequential = false;
  1612         break;
  1613       }
  1614     }
  1615   }
  1616 
  1617   // First check whether the bounds are correct: if (a < LOW || a > HIGH);
  1618   // this check is valid whether or not we emit a sequential-optimized switch.
  1619   cell_t low = cases[0].val;
  1620   if (low != 0) {
  1621     // negate it so we'll get a lower bound of 0.
  1622     low = -low;
  1623     __ lea(tmp, Operand(pri, low));
  1624   } else {
  1625     __ movl(tmp, pri);
  1626   }
  1627 
  1628   cell_t high = abs(cases[0].val - cases[ncases - 1].val);
  1629   __ cmpl(tmp, high);
  1630   __ j(above, defaultCase);
  1631 
  1632   if (sequential) {
  1633     // Optimized table version. The tomfoolery below is because we only have
  1634     // one free register... it seems unlikely pri or alt will be used given
  1635     // that we're at the end of a control-flow point, but we'll play it safe.
  1636     DataLabel table;
  1637     __ push(eax);
  1638     __ movl(eax, &table);
  1639     __ movl(ecx, Operand(eax, ecx, ScaleFour));
  1640     __ pop(eax);
  1641     __ jmp(ecx);
  1642 
  1643     __ bind(&table);
  1644     for (size_t i = 0; i < ncases; i++) {
  1645       Label *label = labelAt(cases[i].offset);
  1646       if (!label)
  1647         return false;
  1648       __ emit_absolute_address(label);
  1649     }
  1650   } else {
  1651     // Slower version. Go through each case and generate a check.
  1652     for (size_t i = 0; i < ncases; i++) {
  1653       Label *label = labelAt(cases[i].offset);
  1654       if (!label)
  1655         return false;
  1656       __ cmpl(pri, cases[i].val);
  1657       __ j(equal, label);
  1658     }
  1659     __ jmp(defaultCase);
  1660   }
  1661   return true;
  1662 }
  1663 
  1664 void
  1665 Compiler::emitErrorPath(Label *dest, int code)
  1666 {
  1667   if (dest->used()) {
  1668     __ bind(dest);
  1669     __ movl(eax, code);
  1670     __ jmp(g_Jit.GetUniversalReturn());
  1671   }
  1672 }
  1673 
  1674 void
  1675 Compiler::emitErrorPaths()
  1676 {
  1677   emitErrorPath(&error_divide_by_zero_, SP_ERROR_DIVIDE_BY_ZERO);
  1678   emitErrorPath(&error_stack_low_, SP_ERROR_STACKLOW);
  1679   emitErrorPath(&error_stack_min_, SP_ERROR_STACKMIN);
  1680   emitErrorPath(&error_bounds_, SP_ERROR_ARRAY_BOUNDS);
  1681   emitErrorPath(&error_memaccess_, SP_ERROR_MEMACCESS);
  1682   emitErrorPath(&error_heap_low_, SP_ERROR_HEAPLOW);
  1683   emitErrorPath(&error_heap_min_, SP_ERROR_HEAPMIN);
  1684   emitErrorPath(&error_integer_overflow_, SP_ERROR_INTEGER_OVERFLOW);
  1685 
  1686   if (extern_error_.used()) {
  1687     __ bind(&extern_error_);
  1688     __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));
  1689     __ movl(eax, Operand(eax, offsetof(sp_context_t, n_err)));
  1690     __ jmp(g_Jit.GetUniversalReturn());
  1691   }
  1692 }
  1693 
  1694 typedef int (*JIT_EXECUTE)(sp_context_t *ctx, uint8_t *memory, void *code);
  1695 
  1696 static void *
  1697 GenerateEntry(void **retp, void **timeoutp)
  1698 {
  1699   AssemblerX86 masm;
  1700 
  1701   __ push(ebp);
  1702   __ movl(ebp, esp);
  1703 
  1704   __ push(esi);   // ebp - 4
  1705   __ push(edi);   // ebp - 8
  1706   __ push(ebx);   // ebp - 12
  1707   __ push(esp);   // ebp - 16
  1708 
  1709   __ movl(ebx, Operand(ebp, 8 + 4 * 0));
  1710   __ movl(eax, Operand(ebp, 8 + 4 * 1));
  1711   __ movl(ecx, Operand(ebp, 8 + 4 * 2));
  1712 
  1713   // Set up run-time registers.
  1714   __ movl(edi, Operand(ebx, offsetof(sp_context_t, sp)));
  1715   __ addl(edi, eax);
  1716   __ movl(esi, eax);
  1717   __ movl(ebx, edi);
  1718 
  1719   // Align the stack.
  1720   __ andl(esp, 0xfffffff0);
  1721 
  1722   // Call into plugin (align the stack first).
  1723   __ call(ecx);
  1724 
  1725   // Get input context, store rval.
  1726   __ movl(ecx, Operand(ebp, 8 + 4 * 0));
  1727   __ movl(Operand(ecx, offsetof(sp_context_t, rval)), pri);
  1728 
  1729   // Set no error.
  1730   __ movl(eax, SP_ERROR_NONE);
  1731 
  1732   // Store latest stk. If we have an error code, we'll jump directly to here,
  1733   // so eax will already be set.
  1734   Label ret;
  1735   __ bind(&ret);
  1736   __ subl(stk, dat);
  1737   __ movl(Operand(ecx, offsetof(sp_context_t, sp)), stk);
  1738 
  1739   // Restore stack.
  1740   __ movl(esp, Operand(ebp, -16));
  1741 
  1742   // Restore registers and gtfo.
  1743   __ pop(ebx);
  1744   __ pop(edi);
  1745   __ pop(esi);
  1746   __ pop(ebp);
  1747   __ ret();
  1748 
  1749   // The universal emergency return will jump to here.
  1750   Label error;
  1751   __ bind(&error);
  1752   __ movl(ecx, Operand(ebp, 8 + 4 * 0)); // ret-path expects ecx = ctx
  1753   __ jmp(&ret);
  1754 
  1755   Label timeout;
  1756   __ bind(&timeout);
  1757   __ movl(eax, SP_ERROR_TIMEOUT);
  1758   __ jmp(&error);
  1759 
  1760   void *code = LinkCode(masm);
  1761   if (!code)
  1762     return NULL;
  1763 
  1764   *retp = reinterpret_cast<uint8_t *>(code) + error.offset();
  1765   *timeoutp = reinterpret_cast<uint8_t *>(code) + timeout.offset();
  1766   return code;
  1767 }
  1768 
  1769 ICompilation *JITX86::ApplyOptions(ICompilation *_IN, ICompilation *_OUT)
  1770 {
  1771   if (_IN == NULL)
  1772     return _OUT;
  1773 
  1774   CompData *_in = (CompData * )_IN;
  1775   CompData *_out = (CompData * )_OUT;
  1776 
  1777   _in->inline_level = _out->inline_level;
  1778   _in->profile = _out->profile;
  1779 
  1780   _out->Abort();
  1781   return _in;
  1782 }
  1783 
  1784 JITX86::JITX86()
  1785 {
  1786   m_pJitEntry = NULL;
  1787 }
  1788 
  1789 bool
  1790 JITX86::InitializeJIT()
  1791 {
  1792   g_pCodeCache = KE_CreateCodeCache();
  1793 
  1794   m_pJitEntry = GenerateEntry(&m_pJitReturn, &m_pJitTimeout);
  1795   if (!m_pJitEntry)
  1796     return false;
  1797 
  1798   MacroAssemblerX86 masm;
  1799   MacroAssemblerX86::GenerateFeatureDetection(masm);
  1800   void *code = LinkCode(masm);
  1801   if (!code)
  1802     return false;
  1803   MacroAssemblerX86::RunFeatureDetection(code);
  1804   KE_FreeCode(g_pCodeCache, code);
  1805 
  1806   return true;
  1807 }
  1808 
  1809 void
  1810 JITX86::ShutdownJIT()
  1811 {
  1812   KE_DestroyCodeCache(g_pCodeCache);
  1813 }
  1814 
  1815 JitFunction *
  1816 JITX86::CompileFunction(BaseRuntime *prt, cell_t pcode_offs, int *err)
  1817 {
  1818   Compiler cc(prt, pcode_offs);
  1819   JitFunction *fun = cc.emit(err);
  1820   if (!fun)
  1821     return NULL;
  1822 
  1823   // Grab the lock before linking code in, since the watchdog timer will look
  1824   // at this list on another thread.
  1825   ke::AutoLock lock(g_Jit.Mutex());
  1826 
  1827   prt->AddJittedFunction(fun);
  1828   return fun;
  1829 }
  1830 
  1831 void
  1832 JITX86::SetupContextVars(BaseRuntime *runtime, BaseContext *pCtx, sp_context_t *ctx)
  1833 {
  1834   ctx->tracker = new tracker_t;
  1835   ctx->tracker->pBase = (ucell_t *)malloc(1024);
  1836   ctx->tracker->pCur = ctx->tracker->pBase;
  1837   ctx->tracker->size = 1024 / sizeof(cell_t);
  1838   ctx->basecx = pCtx;
  1839   ctx->vm[JITVARS_PROFILER] = g_engine2.GetProfiler();
  1840   ctx->plugin = const_cast<sp_plugin_t *>(runtime->plugin());
  1841 }
  1842 
  1843 SPVM_NATIVE_FUNC
  1844 JITX86::CreateFakeNative(SPVM_FAKENATIVE_FUNC callback, void *pData)
  1845 {
  1846   AssemblerX86 masm;
  1847 
  1848   __ push(ebx);
  1849   __ push(edi);
  1850   __ push(esi);
  1851   __ movl(edi, Operand(esp, 16)); // store ctx
  1852   __ movl(esi, Operand(esp, 20)); // store params
  1853   __ movl(ebx, esp);
  1854   __ andl(esp, 0xfffffff0);
  1855   __ subl(esp, 4);
  1856 
  1857   __ push(intptr_t(pData));
  1858   __ push(esi);
  1859   __ push(edi);
  1860   __ call(ExternalAddress((void *)callback));
  1861   __ movl(esp, ebx);
  1862   __ pop(esi);
  1863   __ pop(edi);
  1864   __ pop(ebx);
  1865   __ ret();
  1866 
  1867   return (SPVM_NATIVE_FUNC)LinkCode(masm);
  1868 }
  1869 
  1870 void
  1871 JITX86::DestroyFakeNative(SPVM_NATIVE_FUNC func)
  1872 {
  1873   KE_FreeCode(g_pCodeCache, (void *)func);
  1874 }
  1875 
  1876 ICompilation *
  1877 JITX86::StartCompilation()
  1878 {
  1879   return new CompData;
  1880 }
  1881 
  1882 ICompilation *
  1883 JITX86::StartCompilation(BaseRuntime *runtime)
  1884 {
  1885   return new CompData;
  1886 }
  1887 
  1888 void
  1889 CompData::Abort()
  1890 {
  1891   delete this;
  1892 }
  1893 
  1894 void
  1895 JITX86::FreeContextVars(sp_context_t *ctx)
  1896 {
  1897   free(ctx->tracker->pBase);
  1898   delete ctx->tracker;
  1899 }
  1900 
  1901 bool
  1902 CompData::SetOption(const char *key, const char *val)
  1903 {
  1904   if (strcmp(key, SP_JITCONF_DEBUG) == 0)
  1905     return true;
  1906   if (strcmp(key, SP_JITCONF_PROFILE) == 0) {
  1907     profile = atoi(val);
  1908 
  1909     /** Callbacks must be profiled to profile functions! */
  1910     if ((profile & SP_PROF_FUNCTIONS) == SP_PROF_FUNCTIONS)
  1911       profile |= SP_PROF_CALLBACKS;
  1912 
  1913     return true;
  1914   }
  1915 
  1916   return false;
  1917 }
  1918 
  1919 int
  1920 JITX86::InvokeFunction(BaseRuntime *runtime, JitFunction *fn, cell_t *result)
  1921 {
  1922   sp_context_t *ctx = runtime->GetBaseContext()->GetCtx();
  1923 
  1924   // Note that cip, hp, sp are saved and restored by Execute2().
  1925   ctx->cip = fn->GetPCodeAddress();
  1926 
  1927   JIT_EXECUTE pfn = (JIT_EXECUTE)m_pJitEntry;
  1928 
  1929   if (level_++ == 0)
  1930     frame_id_++;
  1931   int err = pfn(ctx, runtime->plugin()->memory, fn->GetEntryAddress());
  1932   level_--;
  1933 
  1934   *result = ctx->rval;
  1935   return err;
  1936 }
  1937 
  1938 void *
  1939 JITX86::AllocCode(size_t size)
  1940 {
  1941   return Knight::KE_AllocCode(g_pCodeCache, size);
  1942 }
  1943 
  1944 void
  1945 JITX86::FreeCode(void *code)
  1946 {
  1947   KE_FreeCode(g_pCodeCache, code);
  1948 }
  1949 
  1950 void
  1951 JITX86::RegisterRuntime(BaseRuntime *rt)
  1952 {
  1953   mutex_.AssertCurrentThreadOwns();
  1954   runtimes_.append(rt);
  1955 }
  1956 
  1957 void
  1958 JITX86::DeregisterRuntime(BaseRuntime *rt)
  1959 {
  1960   mutex_.AssertCurrentThreadOwns();
  1961   runtimes_.remove(rt);
  1962 }
  1963 
  1964 void
  1965 JITX86::PatchAllJumpsForTimeout()
  1966 {
  1967   mutex_.AssertCurrentThreadOwns();
  1968   for (ke::InlineList<BaseRuntime>::iterator iter = runtimes_.begin(); iter != runtimes_.end(); iter++) {
  1969     BaseRuntime *rt = *iter;
  1970     for (size_t i = 0; i < rt->NumJitFunctions(); i++) {
  1971       JitFunction *fun = rt->GetJitFunction(i);
  1972       uint8_t *base = reinterpret_cast<uint8_t *>(fun->GetEntryAddress());
  1973 
  1974       for (size_t j = 0; j < fun->NumLoopEdges(); j++) {
  1975         const LoopEdge &e = fun->GetLoopEdge(j);
  1976         int32_t diff = intptr_t(m_pJitTimeout) - intptr_t(base + e.offset);
  1977         *reinterpret_cast<int32_t *>(base + e.offset - 4) = diff;
  1978       }
  1979     }
  1980   }
  1981 }
  1982 
  1983 void
  1984 JITX86::UnpatchAllJumpsFromTimeout()
  1985 {
  1986   mutex_.AssertCurrentThreadOwns();
  1987   for (ke::InlineList<BaseRuntime>::iterator iter = runtimes_.begin(); iter != runtimes_.end(); iter++) {
  1988     BaseRuntime *rt = *iter;
  1989     for (size_t i = 0; i < rt->NumJitFunctions(); i++) {
  1990       JitFunction *fun = rt->GetJitFunction(i);
  1991       uint8_t *base = reinterpret_cast<uint8_t *>(fun->GetEntryAddress());
  1992 
  1993       for (size_t j = 0; j < fun->NumLoopEdges(); j++) {
  1994         const LoopEdge &e = fun->GetLoopEdge(j);
  1995         *reinterpret_cast<int32_t *>(base + e.offset - 4) = e.disp32;
  1996       }
  1997     }
  1998   }
  1999 }
sourcemod-central
Feed