sourcemod-central: sourcepawn/jit/x86/jit

     1 /**

     2  * vim: set ts=2 sw=2 tw=99 et:

     3  * =============================================================================

     4  * SourceMod

     5  * Copyright (C) 2004-2008 AlliedModders LLC.  All rights reserved.

     6  * =============================================================================

     7  *

     8  * This program is free software; you can redistribute it and/or modify it under

     9  * the terms of the GNU General Public License, version 3.0, as published by the

    10  * Free Software Foundation.

    11  *

    12  * This program is distributed in the hope that it will be useful, but WITHOUT

    13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    14  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more

    15  * details.

    16  *

    17  * You should have received a copy of the GNU General Public License along with

    18  * this program.  If not, see <http://www.gnu.org/licenses/>.

    19  *

    20  * As a special exception, AlliedModders LLC gives you permission to link the

    21  * code of this program (as well as its derivative works) to "Half-Life 2," the

    22  * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software

    23  * by the Valve Corporation.  You must obey the GNU General Public License in

    24  * all respects for all other code used.  Additionally, AlliedModders LLC grants

    25  * this exception to all derivative works.  AlliedModders LLC defines further

    26  * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),

    27  * or <http://www.sourcemod.net/license.php>.

    28  *

    29  * Version: $Id$

    30  */

    32 #include <string.h>

    33 #include <stdlib.h>

    34 #include <assert.h>

    35 #include "jit_x86.h"

    36 #include "../sp_vm_engine.h"

    37 #include "../engine2.h"

    38 #include "../BaseRuntime.h"

    39 #include "../sp_vm_basecontext.h"

    40 #include "watchdog_timer.h"

    41 #include "interpreter.h"

    43 using namespace Knight;

    45 #if defined USE_UNGEN_OPCODES

    46 #include "ungen_opcodes.h"

    47 #endif

    49 #define __ masm.

    51 JITX86 g_Jit;

    52 KeCodeCache *g_pCodeCache = NULL;

    53 ISourcePawnEngine *engine = &g_engine1;

    55 static inline uint8_t *

    56 LinkCode(AssemblerX86 &masm)

    57 {

    58   if (masm.outOfMemory())

    59     return NULL;

    61   void *code = Knight::KE_AllocCode(g_pCodeCache, masm.length());

    62   if (!code)

    63     return NULL;

    65   masm.emitToExecutableMemory(code);

    66   return reinterpret_cast<uint8_t *>(code);

    67 }

    69 static inline ConditionCode

    70 OpToCondition(OPCODE op)

    71 {

    72   switch (op) {

    73    case OP_EQ:

    74    case OP_JEQ:

    75     return equal;

    76    case OP_NEQ:

    77    case OP_JNEQ:

    78     return not_equal;

    79    case OP_SLESS:

    80    case OP_JSLESS:

    81     return less;

    82    case OP_SLEQ:

    83    case OP_JSLEQ:

    84     return less_equal;

    85    case OP_SGRTR:

    86    case OP_JSGRTR:

    87     return greater;

    88    case OP_SGEQ:

    89    case OP_JSGEQ:

    90     return greater_equal;

    91    default:

    92     assert(false);

    93     return negative;

    94   }

    95 }

    97 struct array_creation_t

    98 {

    99   const cell_t *dim_list;     /* Dimension sizes */

   100   cell_t dim_count;           /* Number of dimensions */

   101   cell_t *data_offs;          /* Current offset AFTER the indirection vectors (data) */

   102   cell_t *base;               /* array base */

   103 };

   105 static cell_t

   106 GenerateInnerArrayIndirectionVectors(array_creation_t *ar, int dim, cell_t cur_offs)

   107 {

   108   cell_t write_offs = cur_offs;

   109   cell_t *data_offs = ar->data_offs;

   111   cur_offs += ar->dim_list[dim];

   113   // Dimension n-x where x > 2 will have sub-vectors.

   114   // Otherwise, we just need to reference the data section.

   115   if (ar->dim_count > 2 && dim < ar->dim_count - 2) {

   116     // For each index at this dimension, write offstes to our sub-vectors.

   117     // After we write one sub-vector, we generate its sub-vectors recursively.

   118     // At the end, we're given the next offset we can use.

   119     for (int i = 0; i < ar->dim_list[dim]; i++) {

   120       ar->base[write_offs] = (cur_offs - write_offs) * sizeof(cell_t);

   121       write_offs++;

   122       cur_offs = GenerateInnerArrayIndirectionVectors(ar, dim + 1, cur_offs);

   123     }

   124   } else {

   125     // In this section, there are no sub-vectors, we need to write offsets

   126     // to the data.  This is separate so the data stays in one big chunk.

   127     // The data offset will increment by the size of the last dimension,

   128     // because that is where the data is finally computed as.

   129     for (int i = 0; i < ar->dim_list[dim]; i++) {

   130       ar->base[write_offs] = (*data_offs - write_offs) * sizeof(cell_t);

   131       write_offs++;

   132       *data_offs = *data_offs + ar->dim_list[dim + 1];

   133     }

   134   }

   136   return cur_offs;

   137 }

   139 static cell_t

   140 calc_indirection(const array_creation_t *ar, cell_t dim)

   141 {

   142   cell_t size = ar->dim_list[dim];

   143   if (dim < ar->dim_count - 2)

   144     size += ar->dim_list[dim] * calc_indirection(ar, dim + 1);

   145   return size;

   146 }

   148 static cell_t

   149 GenerateArrayIndirectionVectors(cell_t *arraybase, cell_t dims[], cell_t _dimcount, bool autozero)

   150 {

   151   array_creation_t ar;

   152   cell_t data_offs;

   154   /* Reverse the dimensions */

   155   cell_t dim_list[sDIMEN_MAX];

   156   int cur_dim = 0;

   157   for (int i = _dimcount - 1; i >= 0; i--)

   158     dim_list[cur_dim++] = dims[i];

   160   ar.base = arraybase;

   161   ar.dim_list = dim_list;

   162   ar.dim_count = _dimcount;

   163   ar.data_offs = &data_offs;

   165   data_offs = calc_indirection(&ar, 0);

   166   GenerateInnerArrayIndirectionVectors(&ar, 0, 0);

   167   return data_offs;

   168 }

   170 int

   171 GenerateFullArray(BaseRuntime *rt, uint32_t argc, cell_t *argv, int autozero)

   172 {

   173   sp_context_t *ctx = rt->GetBaseContext()->GetCtx();

   175   // Calculate how many cells are needed.

   176   if (argv[0] <= 0)

   177     return SP_ERROR_ARRAY_TOO_BIG;

   179   uint32_t cells = argv[0];

   181   for (uint32_t dim = 1; dim < argc; dim++) {

   182     cell_t dimsize = argv[dim];

   183     if (dimsize <= 0)

   184       return SP_ERROR_ARRAY_TOO_BIG;

   185     if (!ke::IsUint32MultiplySafe(cells, dimsize))

   186       return SP_ERROR_ARRAY_TOO_BIG;

   187     cells *= uint32_t(dimsize);

   188     if (!ke::IsUint32AddSafe(cells, dimsize))

   189       return SP_ERROR_ARRAY_TOO_BIG;

   190     cells += uint32_t(dimsize);

   191   }

   193   if (!ke::IsUint32MultiplySafe(cells, 4))

   194     return SP_ERROR_ARRAY_TOO_BIG;

   196   uint32_t bytes = cells * 4;

   197   if (!ke::IsUint32AddSafe(ctx->hp, bytes))

   198     return SP_ERROR_ARRAY_TOO_BIG;

   200   uint32_t new_hp = ctx->hp + bytes;

   201   cell_t *dat_hp = reinterpret_cast<cell_t *>(rt->plugin()->memory + new_hp);

   203   // argv, coincidentally, is STK.

   204   if (dat_hp >= argv - STACK_MARGIN)

   205     return SP_ERROR_HEAPLOW;

   207   if (int err = PushTracker(rt->GetBaseContext()->GetCtx(), bytes))

   208     return err;

   210   cell_t *base = reinterpret_cast<cell_t *>(rt->plugin()->memory + ctx->hp);

   211   cell_t offs = GenerateArrayIndirectionVectors(base, argv, argc, !!autozero);

   212   assert(size_t(offs) == cells);

   214   argv[argc - 1] = ctx->hp;

   215   ctx->hp = new_hp;

   216   return SP_ERROR_NONE;

   217 }

   219 #if !defined NDEBUG

   220 static const char *

   221 GetFunctionName(const sp_plugin_t *plugin, uint32_t offs)

   222 {

   223   if (!plugin->debug.unpacked) {

   224     uint32_t max, iter;

   225     sp_fdbg_symbol_t *sym;

   226     sp_fdbg_arraydim_t *arr;

   227     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   229     max = plugin->debug.syms_num;

   230     for (iter = 0; iter < max; iter++) {

   231       sym = (sp_fdbg_symbol_t *)cursor;

   233       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   234         return plugin->debug.stringbase + sym->name;

   236       if (sym->dimcount > 0) {

   237         cursor += sizeof(sp_fdbg_symbol_t);

   238         arr = (sp_fdbg_arraydim_t *)cursor;

   239         cursor += sizeof(sp_fdbg_arraydim_t) * sym->dimcount;

   240         continue;

   241       }

   243       cursor += sizeof(sp_fdbg_symbol_t);

   244     }

   245   } else {

   246     uint32_t max, iter;

   247     sp_u_fdbg_symbol_t *sym;

   248     sp_u_fdbg_arraydim_t *arr;

   249     uint8_t *cursor = (uint8_t *)(plugin->debug.symbols);

   251     max = plugin->debug.syms_num;

   252     for (iter = 0; iter < max; iter++) {

   253       sym = (sp_u_fdbg_symbol_t *)cursor;

   255       if (sym->ident == SP_SYM_FUNCTION && sym->codestart <= offs && sym->codeend > offs)

   256         return plugin->debug.stringbase + sym->name;

   258       if (sym->dimcount > 0) {

   259         cursor += sizeof(sp_u_fdbg_symbol_t);

   260         arr = (sp_u_fdbg_arraydim_t *)cursor;

   261         cursor += sizeof(sp_u_fdbg_arraydim_t) * sym->dimcount;

   262         continue;

   263       }

   265       cursor += sizeof(sp_u_fdbg_symbol_t);

   266     }

   267   }

   269   return NULL;

   270 }

   271 #endif

   273 static int

   274 CompileFromThunk(BaseRuntime *runtime, cell_t pcode_offs, void **addrp, char *pc)

   275 {

   276   // If the watchdog timer has declared a timeout, we must process it now,

   277   // and possibly refuse to compile, since otherwise we will compile a

   278   // function that is not patched for timeouts.

   279   if (!g_WatchdogTimer.HandleInterrupt())

   280     return SP_ERROR_TIMEOUT;

   282   JitFunction *fn = runtime->GetJittedFunctionByOffset(pcode_offs);

   283   if (!fn) {

   284     int err;

   285     fn = g_Jit.CompileFunction(runtime, pcode_offs, &err);

   286     if (!fn)

   287       return err;

   288   }

   290 #if !defined NDEBUG

   291   g_engine1.GetDebugHook()->OnDebugSpew(

   292       "Patching thunk to %s::%s\n",

   293       runtime->plugin()->name,

   294       GetFunctionName(runtime->plugin(), pcode_offs));

   295 #endif

   297   *addrp = fn->GetEntryAddress();

   299   /* Right now, we always keep the code RWE */

   300   *(intptr_t *)(pc - 4) = intptr_t(fn->GetEntryAddress()) - intptr_t(pc);

   301   return SP_ERROR_NONE;

   302 }

   304 Compiler::Compiler(BaseRuntime *rt, cell_t pcode_offs)

   305   : rt_(rt),

   306     plugin_(rt->plugin()),

   307     error_(SP_ERROR_NONE),

   308     pcode_start_(pcode_offs),

   309     code_start_(reinterpret_cast<cell_t *>(plugin_->pcode + pcode_start_)),

   310     cip_(code_start_),

   311     code_end_(reinterpret_cast<cell_t *>(plugin_->pcode + plugin_->pcode_size))

   312 {

   313   size_t nmaxops = plugin_->pcode_size / sizeof(cell_t) + 1;

   314   jump_map_ = new Label[nmaxops];

   315 }

   317 Compiler::~Compiler()

   318 {

   319   delete [] jump_map_;

   320 }

   322 JitFunction *

   323 Compiler::emit(int *errp)

   324 {

   325   if (cip_ >= code_end_ || *cip_ != OP_PROC) {

   326     *errp = SP_ERROR_INVALID_INSTRUCTION;

   327     return NULL;

   328   }

   330 #if !defined NDEBUG

   331   g_engine1.GetDebugHook()->OnDebugSpew(

   332       "Compiling function %s::%s\n",

   333       plugin_->name,

   334       GetFunctionName(plugin_, pcode_start_));

   336 #endif

   337 #if defined DEBUG

   338   SpewOpcode(plugin_, code_start_, cip_);

   339 #endif

   341   cell_t *codeseg = reinterpret_cast<cell_t *>(plugin_->pcode);

   343   cip_++;

   344   if (!emitOp(OP_PROC)) {

   345       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   346       return NULL;

   347   }

   349   while (cip_ < code_end_) {

   350     // If we reach the end of this function, or the beginning of a new

   351     // procedure, then stop.

   352     if (*cip_ == OP_PROC || *cip_ == OP_ENDPROC)

   353       break;

   355 #if defined DEBUG

   356     SpewOpcode(plugin_, code_start_, cip_);

   357 #endif

   359     // We assume every instruction is a jump target, so before emitting

   360     // an opcode, we bind its corresponding label.

   361     __ bind(&jump_map_[cip_ - codeseg]);

   363     OPCODE op = (OPCODE)readCell();

   364     if (!emitOp(op) || error_ != SP_ERROR_NONE) {

   365       *errp = (error_ == SP_ERROR_NONE) ? SP_ERROR_OUT_OF_MEMORY : error_;

   366       return NULL;

   367     }

   368   }

   370   emitCallThunks();

   371   emitErrorPaths();

   373   uint8_t *code = LinkCode(masm);

   374   if (!code) {

   375     *errp = SP_ERROR_OUT_OF_MEMORY;

   376     return NULL;

   377   }

   379   LoopEdge *edges = new LoopEdge[backward_jumps_.length()];

   380   for (size_t i = 0; i < backward_jumps_.length(); i++) {

   381     edges[i].offset = backward_jumps_[i];

   382     edges[i].disp32 = *reinterpret_cast<int32_t *>(code + edges[i].offset - 4);

   383   }

   385   return new JitFunction(code, pcode_start_, edges, backward_jumps_.length());

   386 }

   388 bool

   389 Compiler::emitOp(OPCODE op)

   390 {

   391   switch (op) {

   392     case OP_MOVE_PRI:

   393       __ movl(pri, alt);

   394       break;

   396     case OP_MOVE_ALT:

   397       __ movl(alt, pri);

   398       break;

   400     case OP_XCHG:

   401       __ xchgl(pri, alt);

   402       break;

   404     case OP_ZERO:

   405     {

   406       cell_t offset = readCell();

   407       __ movl(Operand(dat, offset), 0);

   408       break;

   409     }

   411     case OP_ZERO_S:

   412     {

   413       cell_t offset = readCell();

   414       __ movl(Operand(frm, offset), 0);

   415       break;

   416     }

   418     case OP_PUSH_PRI:

   419     case OP_PUSH_ALT:

   420     {

   421       Register reg = (op == OP_PUSH_PRI) ? pri : alt;

   422       __ movl(Operand(stk, -4), reg);

   423       __ subl(stk, 4);

   424       break;

   425     }

   427     case OP_PUSH_C:

   428     case OP_PUSH2_C:

   429     case OP_PUSH3_C:

   430     case OP_PUSH4_C:

   431     case OP_PUSH5_C:

   432     {

   433       int n = 1;

   434       if (op >= OP_PUSH2_C)

   435         n = ((op - OP_PUSH2_C) / 4) + 2;

   437       int i = 1;

   438       do {

   439         cell_t val = readCell();

   440         __ movl(Operand(stk, -(4 * i)), val);

   441       } while (i++ < n);

   442       __ subl(stk, 4 * n);

   443       break;

   444     }

   446     case OP_PUSH_ADR:

   447     case OP_PUSH2_ADR:

   448     case OP_PUSH3_ADR:

   449     case OP_PUSH4_ADR:

   450     case OP_PUSH5_ADR:

   451     {

   452       int n = 1;

   453       if (op >= OP_PUSH2_ADR)

   454         n = ((op - OP_PUSH2_ADR) / 4) + 2;

   456       int i = 1;

   458       // We temporarily relocate FRM to be a local address instead of an

   459       // absolute address.

   460       __ subl(frm, dat);

   461       do {

   462         cell_t offset = readCell();

   463         __ lea(tmp, Operand(frm, offset));

   464         __ movl(Operand(stk, -(4 * i)), tmp);

   465       } while (i++ < n);

   466       __ subl(stk, 4 * n);

   467       __ addl(frm, dat);

   468       break;

   469     }

   471     case OP_PUSH_S:

   472     case OP_PUSH2_S:

   473     case OP_PUSH3_S:

   474     case OP_PUSH4_S:

   475     case OP_PUSH5_S:

   476     {

   477       int n = 1;

   478       if (op >= OP_PUSH2_S)

   479         n = ((op - OP_PUSH2_S) / 4) + 2;

   481       int i = 1;

   482       do {

   483         cell_t offset = readCell();

   484         __ movl(tmp, Operand(frm, offset));

   485         __ movl(Operand(stk, -(4 * i)), tmp);

   486       } while (i++ < n);

   487       __ subl(stk, 4 * n);

   488       break;

   489     }

   491     case OP_PUSH:

   492     case OP_PUSH2:

   493     case OP_PUSH3:

   494     case OP_PUSH4:

   495     case OP_PUSH5:

   496     {

   497       int n = 1;

   498       if (op >= OP_PUSH2)

   499         n = ((op - OP_PUSH2) / 4) + 2;

   501       int i = 1;

   502       do {

   503         cell_t offset = readCell();

   504         __ movl(tmp, Operand(dat, offset));

   505         __ movl(Operand(stk, -(4 * i)), tmp);

   506       } while (i++ < n);

   507       __ subl(stk, 4 * n);

   508       break;

   509     }

   511     case OP_ZERO_PRI:

   512       __ xorl(pri, pri);

   513       break;

   515     case OP_ZERO_ALT:

   516       __ xorl(alt, alt);

   517       break;

   519     case OP_ADD:

   520       __ addl(pri, alt);

   521       break;

   523     case OP_SUB:

   524       __ subl(pri, alt);

   525       break;

   527     case OP_SUB_ALT:

   528       __ movl(tmp, alt);

   529       __ subl(tmp, pri);

   530       __ movl(pri, tmp);

   531       break;

   533     case OP_PROC:

   534       // Push the old frame onto the stack.

   535       __ movl(tmp, Operand(frmAddr()));

   536       __ movl(Operand(stk, -4), tmp);

   537       __ subl(stk, 8);    // extra unused slot for non-existant CIP

   539       // Get and store the new frame.

   540       __ movl(tmp, stk);

   541       __ movl(frm, stk);

   542       __ subl(tmp, dat);

   543       __ movl(Operand(frmAddr()), tmp);

   545       // Align the stack to 16-bytes (each call adds 4 bytes).

   546       __ subl(esp, 12);

   547       break;

   549     case OP_IDXADDR_B:

   550     {

   551       cell_t val = readCell();

   552       __ shll(pri, val);

   553       __ addl(pri, alt);

   554       break;

   555     }

   557     case OP_SHL:

   558       __ movl(ecx, alt);

   559       __ shll_cl(pri);

   560       break;

   562     case OP_SHR:

   563       __ movl(ecx, alt);

   564       __ shrl_cl(pri);

   565       break;

   567     case OP_SSHR:

   568       __ movl(ecx, alt);

   569       __ sarl_cl(pri);

   570       break;

   572     case OP_SHL_C_PRI:

   573     case OP_SHL_C_ALT:

   574     {

   575       Register reg = (op == OP_SHL_C_PRI) ? pri : alt;

   576       cell_t val = readCell();

   577       __ shll(reg, val);

   578       break;

   579     }

   581     case OP_SHR_C_PRI:

   582     case OP_SHR_C_ALT:

   583     {

   584       Register reg = (op == OP_SHR_C_PRI) ? pri : alt;

   585       cell_t val = readCell();

   586       __ shrl(reg, val);

   587       break;

   588     }

   590     case OP_SMUL:

   591       __ imull(pri, alt);

   592       break;

   594     case OP_NOT:

   595       __ testl(eax, eax);

   596       __ movl(eax, 0);

   597       __ set(zero, r8_al);

   598       break;

   600     case OP_NEG:

   601       __ negl(eax);

   602       break;

   604     case OP_XOR:

   605       __ xorl(pri, alt);

   606       break;

   608     case OP_OR:

   609       __ orl(pri, alt);

   610       break;

   612     case OP_AND:

   613       __ andl(pri, alt);

   614       break;

   616     case OP_INVERT:

   617       __ notl(pri);

   618       break;

   620     case OP_ADD_C:

   621     {

   622       cell_t val = readCell();

   623       __ addl(pri, val);

   624       break;

   625     }

   627     case OP_SMUL_C:

   628     {

   629       cell_t val = readCell();

   630       __ imull(pri, pri, val);

   631       break;

   632     }

   634     case OP_EQ:

   635     case OP_NEQ:

   636     case OP_SLESS:

   637     case OP_SLEQ:

   638     case OP_SGRTR:

   639     case OP_SGEQ:

   640     {

   641       ConditionCode cc = OpToCondition(op);

   642       __ cmpl(pri, alt);

   643       __ movl(pri, 0);

   644       __ set(cc, r8_al);

   645       break;

   646     }

   648     case OP_EQ_C_PRI:

   649     case OP_EQ_C_ALT:

   650     {

   651       Register reg = (op == OP_EQ_C_PRI) ? pri : alt;

   652       cell_t val = readCell();

   653       __ cmpl(reg, val);

   654       __ movl(pri, 0);

   655       __ set(equal, r8_al);

   656       break;

   657     }

   659     case OP_INC_PRI:

   660     case OP_INC_ALT:

   661     {

   662       Register reg = (OP_INC_PRI) ? pri : alt;

   663       __ addl(reg, 1);

   664       break;

   665     }

   667     case OP_INC:

   668     case OP_INC_S:

   669     {

   670       Register base = (op == OP_INC) ? dat : frm;

   671       cell_t offset = readCell();

   672       __ addl(Operand(base, offset), 1);

   673       break;

   674     }

   676     case OP_INC_I:

   677       __ addl(Operand(dat, pri, NoScale), 1);

   678       break;

   680     case OP_DEC_PRI:

   681     case OP_DEC_ALT:

   682     {

   683       Register reg = (op == OP_DEC_PRI) ? pri : alt;

   684       __ subl(reg, 1);

   685       break;

   686     }

   688     case OP_DEC:

   689     case OP_DEC_S:

   690     {

   691       Register base = (op == OP_DEC) ? dat : frm;

   692       cell_t offset = readCell();

   693       __ subl(Operand(base, offset), 1);

   694       break;

   695     }

   697     case OP_DEC_I:

   698       __ subl(Operand(dat, pri, NoScale), 1);

   699       break;

   701     case OP_LOAD_PRI:

   702     case OP_LOAD_ALT:

   703     {

   704       Register reg = (op == OP_LOAD_PRI) ? pri : alt;

   705       cell_t offset = readCell();

   706       __ movl(reg, Operand(dat, offset));

   707       break;

   708     }

   710     case OP_LOAD_BOTH:

   711     {

   712       cell_t offs1 = readCell();

   713       cell_t offs2 = readCell();

   714       __ movl(pri, Operand(dat, offs1));

   715       __ movl(alt, Operand(dat, offs2));

   716       break;

   717     }

   719     case OP_LOAD_S_PRI:

   720     case OP_LOAD_S_ALT:

   721     {

   722       Register reg = (op == OP_LOAD_S_PRI) ? pri : alt;

   723       cell_t offset = readCell();

   724       __ movl(reg, Operand(frm, offset));

   725       break;

   726     }

   728     case OP_LOAD_S_BOTH:

   729     {

   730       cell_t offs1 = readCell();

   731       cell_t offs2 = readCell();

   732       __ movl(pri, Operand(frm, offs1));

   733       __ movl(alt, Operand(frm, offs2));

   734       break;

   735     }

   737     case OP_LREF_S_PRI:

   738     case OP_LREF_S_ALT:

   739     {

   740       Register reg = (op == OP_LREF_S_PRI) ? pri : alt;

   741       cell_t offset = readCell();

   742       __ movl(reg, Operand(frm, offset));

   743       __ movl(reg, Operand(dat, reg, NoScale));

   744       break;

   745     }

   747     case OP_CONST_PRI:

   748     case OP_CONST_ALT:

   749     {

   750       Register reg = (op == OP_CONST_PRI) ? pri : alt;

   751       cell_t val = readCell();

   752       __ movl(reg, val);

   753       break;

   754     }

   756     case OP_ADDR_PRI:

   757     case OP_ADDR_ALT:

   758     {

   759       Register reg = (op == OP_ADDR_PRI) ? pri : alt;

   760       cell_t offset = readCell();

   761       __ movl(reg, Operand(frmAddr()));

   762       __ addl(reg, offset);

   763       break;

   764     }

   766     case OP_STOR_PRI:

   767     case OP_STOR_ALT:

   768     {

   769       Register reg = (op == OP_STOR_PRI) ? pri : alt;

   770       cell_t offset = readCell();

   771       __ movl(Operand(dat, offset), reg);

   772       break;

   773     }

   775     case OP_STOR_S_PRI:

   776     case OP_STOR_S_ALT:

   777     {

   778       Register reg = (op == OP_STOR_S_PRI) ? pri : alt;

   779       cell_t offset = readCell();

   780       __ movl(Operand(frm, offset), reg);

   781       break;

   782     }

   784     case OP_IDXADDR:

   785       __ lea(pri, Operand(alt, pri, ScaleFour));

   786       break;

   788     case OP_SREF_S_PRI:

   789     case OP_SREF_S_ALT:

   790     {

   791       Register reg = (op == OP_SREF_S_PRI) ? pri : alt;

   792       cell_t offset = readCell();

   793       __ movl(tmp, Operand(frm, offset));

   794       __ movl(Operand(dat, tmp, NoScale), reg);

   795       break;

   796     }

   798     case OP_POP_PRI:

   799     case OP_POP_ALT:

   800     {

   801       Register reg = (op == OP_POP_PRI) ? pri : alt;

   802       __ movl(reg, Operand(stk, 0));

   803       __ addl(stk, 4);

   804       break;

   805     }

   807     case OP_SWAP_PRI:

   808     case OP_SWAP_ALT:

   809     {

   810       Register reg = (op == OP_SWAP_PRI) ? pri : alt;

   811       __ movl(tmp, Operand(stk, 0));

   812       __ movl(Operand(stk, 0), reg);

   813       __ movl(reg, tmp);

   814       break;

   815     }

   817     case OP_LIDX:

   818       __ lea(pri, Operand(alt, pri, ScaleFour));

   819       __ movl(pri, Operand(dat, pri, NoScale));

   820       break;

   822     case OP_LIDX_B:

   823     {

   824       cell_t val = readCell();

   825       if (val >= 0 && val <= 3) {

   826         __ lea(pri, Operand(alt, pri, Scale(val)));

   827       } else {

   828         __ shll(pri, val);

   829         __ addl(pri, alt);

   830       }

   831       emitCheckAddress(pri);

   832       __ movl(pri, Operand(dat, pri, NoScale));

   833       break;

   834     }

   836     case OP_CONST:

   837     case OP_CONST_S:

   838     {

   839       Register base = (op == OP_CONST) ? dat : frm;

   840       cell_t offset = readCell();

   841       cell_t val = readCell();

   842       __ movl(Operand(base, offset), val);

   843       break;

   844     }

   846     case OP_LOAD_I:

   847       emitCheckAddress(pri);

   848       __ movl(pri, Operand(dat, pri, NoScale));

   849       break;

   851     case OP_STOR_I:

   852       emitCheckAddress(alt);

   853       __ movl(Operand(dat, alt, NoScale), pri);

   854       break;

   856     case OP_SDIV:

   857     case OP_SDIV_ALT:

   858     {

   859       Register dividend = (op == OP_SDIV) ? pri : alt;

   860       Register divisor = (op == OP_SDIV) ? alt : pri;

   862       // Guard against divide-by-zero.

   863       __ testl(divisor, divisor);

   864       __ j(zero, &error_divide_by_zero_);

   866       // A more subtle case; -INT_MIN / -1 yields an overflow exception.

   867       Label ok;

   868       __ cmpl(divisor, -1);

   869       __ j(not_equal, &ok);

   870       __ cmpl(dividend, 0x80000000);

   871       __ j(equal, &error_integer_overflow_);

   872       __ bind(&ok);

   874       // Now we can actually perform the divide.

   875       __ movl(tmp, divisor);

   876       if (op == OP_SDIV)

   877         __ movl(edx, dividend);

   878       else

   879         __ movl(eax, dividend);

   880       __ sarl(edx, 31);

   881       __ idivl(tmp);

   882       break;

   883     }

   885     case OP_LODB_I:

   886     {

   887       cell_t val = readCell();

   888       emitCheckAddress(pri);

   889       __ movl(pri, Operand(dat, pri, NoScale));

   890       if (val == 1)

   891         __ andl(pri, 0xff);

   892       else if (val == 2)

   893         __ andl(pri, 0xffff);

   894       break;

   895     }

   897     case OP_STRB_I:

   898     {

   899       cell_t val = readCell();

   900       emitCheckAddress(alt);

   901       if (val == 1)

   902         __ movb(Operand(dat, alt, NoScale), pri);

   903       else if (val == 2)

   904         __ movw(Operand(dat, alt, NoScale), pri);

   905       else if (val == 4)

   906         __ movl(Operand(dat, alt, NoScale), pri);

   907       break;

   908     }

   910     case OP_RETN:

   911     {

   912       // Restore the old frame pointer.

   913       __ movl(frm, Operand(stk, 4));              // get the old frm

   914       __ addl(stk, 8);                            // pop stack

   915       __ movl(Operand(frmAddr()), frm);           // store back old frm

   916       __ addl(frm, dat);                          // relocate

   918       // Remove parameters.

   919       __ movl(tmp, Operand(stk, 0));

   920       __ lea(stk, Operand(stk, tmp, ScaleFour, 4));

   921       __ addl(esp, 12);

   922       __ ret();

   923       break;

   924     }

   926     case OP_MOVS:

   927     {

   928       cell_t val = readCell();

   929       unsigned dwords = val / 4;

   930       unsigned bytes = val % 4;

   932       __ cld();

   933       __ push(esi);

   934       __ push(edi);

   935       // Note: set edi first, since we need esi.

   936       __ lea(edi, Operand(dat, alt, NoScale));

   937       __ lea(esi, Operand(dat, pri, NoScale));

   938       if (dwords) {

   939         __ movl(ecx, dwords);

   940         __ rep_movsd();

   941       }

   942       if (bytes) {

   943         __ movl(ecx, bytes);

   944         __ rep_movsb();

   945       }

   946       __ pop(edi);

   947       __ pop(esi);

   948       break;

   949     }

   951     case OP_FILL:

   952     {

   953       // eax/pri is used implicitly.

   954       unsigned dwords = readCell() / 4;

   955       __ push(edi);

   956       __ lea(edi, Operand(dat, alt, NoScale));

   957       __ movl(ecx, dwords);

   958       __ cld();

   959       __ rep_stosd();

   960       __ pop(edi);

   961       break;

   962     }

   964     case OP_STRADJUST_PRI:

   965       __ addl(pri, 4);

   966       __ sarl(pri, 2);

   967       break;

   969     case OP_FABS:

   970       __ movl(pri, Operand(stk, 0));

   971       __ andl(pri, 0x7fffffff);

   972       __ addl(stk, 4);

   973       break;

   975     case OP_FLOAT:

   976       if (MacroAssemblerX86::Features().sse2) {

   977         __ cvtsi2ss(xmm0, Operand(edi, 0));

   978         __ movd(pri, xmm0);

   979       } else {

   980         __ fild32(Operand(edi, 0));

   981         __ subl(esp, 4);

   982         __ fstp32(Operand(esp, 0));

   983         __ pop(pri);

   984       }

   985       __ addl(stk, 4);

   986       break;

   988     case OP_FLOATADD:

   989     case OP_FLOATSUB:

   990     case OP_FLOATMUL:

   991     case OP_FLOATDIV:

   992       if (MacroAssemblerX86::Features().sse2) {

   993         __ movss(xmm0, Operand(stk, 0));

   994         if (op == OP_FLOATADD)

   995           __ addss(xmm0, Operand(stk, 4));

   996         else if (op == OP_FLOATSUB)

   997           __ subss(xmm0, Operand(stk, 4));

   998         else if (op == OP_FLOATMUL)

   999           __ mulss(xmm0, Operand(stk, 4));

  1000         else if (op == OP_FLOATDIV)

  1001           __ divss(xmm0, Operand(stk, 4));

  1002         __ movd(pri, xmm0);

  1003       } else {

  1004         __ subl(esp, 4);

  1005         __ fld32(Operand(stk, 0));

  1007         if (op == OP_FLOATADD)

  1008           __ fadd32(Operand(stk, 4));

  1009         else if (op == OP_FLOATSUB)

  1010           __ fsub32(Operand(stk, 4));

  1011         else if (op == OP_FLOATMUL)

  1012           __ fmul32(Operand(stk, 4));

  1013         else if (op == OP_FLOATDIV)

  1014           __ fdiv32(Operand(stk, 4));

  1016         __ fstp32(Operand(esp, 0));

  1017         __ pop(pri);

  1018       }

  1019       __ addl(stk, 8);

  1020       break;

  1022     case OP_RND_TO_NEAREST:

  1023     {

  1024       if (MacroAssemblerX86::Features().sse) {

  1025         // Assume no one is touching MXCSR.

  1026         __ cvtss2si(pri, Operand(stk, 0));

  1027       } else {

  1028         static float kRoundToNearest = 0.5f;

  1029         // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1030         __ fld32(Operand(stk, 0));

  1031         __ fadd32(st0, st0);

  1032         __ fadd32(Operand(ExternalAddress(&kRoundToNearest)));

  1033         __ subl(esp, 4);

  1034         __ fistp32(Operand(esp, 0));

  1035         __ pop(pri);

  1036         __ sarl(pri, 1);

  1037       }

  1038       __ addl(stk, 4);

  1039       break;

  1040     }

  1042     case OP_RND_TO_CEIL:

  1043     {

  1044       static float kRoundToCeil = -0.5f;

  1045       // From http://wurstcaptures.untergrund.net/assembler_tricks.html#fastfloorf

  1046       __ fld32(Operand(stk, 0));

  1047       __ fadd32(st0, st0);

  1048       __ fsubr32(Operand(ExternalAddress(&kRoundToCeil)));

  1049       __ subl(esp, 4);

  1050       __ fistp32(Operand(esp, 0));

  1051       __ pop(pri);

  1052       __ sarl(pri, 1);

  1053       __ negl(pri);

  1054       __ addl(stk, 4);

  1055       break;

  1056     }

  1058     case OP_RND_TO_ZERO:

  1059       if (MacroAssemblerX86::Features().sse) {

  1060         __ cvttss2si(pri, Operand(stk, 0));

  1061       } else {

  1062         __ fld32(Operand(stk, 0));

  1063         __ subl(esp, 8);

  1064         __ fstcw(Operand(esp, 4));

  1065         __ movl(Operand(esp, 0), 0xfff);

  1066         __ fldcw(Operand(esp, 0));

  1067         __ fistp32(Operand(esp, 0));

  1068         __ pop(pri);

  1069         __ fldcw(Operand(esp, 0));

  1070         __ addl(esp, 4);

  1071       }

  1072       __ addl(stk, 4);

  1073       break;

  1075     case OP_RND_TO_FLOOR:

  1076       __ fld32(Operand(stk, 0));

  1077       __ subl(esp, 8);

  1078       __ fstcw(Operand(esp, 4));

  1079       __ movl(Operand(esp, 0), 0x7ff);

  1080       __ fldcw(Operand(esp, 0));

  1081       __ fistp32(Operand(esp, 0));

  1082       __ pop(eax);

  1083       __ fldcw(Operand(esp, 0));

  1084       __ addl(esp, 4);

  1085       __ addl(stk, 4);

  1086       break;

  1088     // This is the old float cmp, which returns ordered results. In newly

  1089     // compiled code it should not be used or generated.

  1090     //

  1091     // Note that the checks here are inverted: the test is |rhs OP lhs|.

  1092     case OP_FLOATCMP:

  1093     {

  1094       Label bl, ab, done;

  1095       if (MacroAssemblerX86::Features().sse) {

  1096         __ movss(xmm0, Operand(stk, 4));

  1097         __ ucomiss(Operand(stk, 0), xmm0);

  1098       } else {

  1099         __ fld32(Operand(stk, 0));

  1100         __ fld32(Operand(stk, 4));

  1101         __ fucomip(st1);

  1102         __ fstp(st0);

  1103       }

  1104       __ j(above, &ab);

  1105       __ j(below, &bl);

  1106       __ xorl(pri, pri);

  1107       __ jmp(&done);

  1108       __ bind(&ab);

  1109       __ movl(pri, -1);

  1110       __ jmp(&done);

  1111       __ bind(&bl);

  1112       __ movl(pri, 1);

  1113       __ bind(&done);

  1114       __ addl(stk, 8);

  1115       break;

  1116     }

  1118     case OP_FLOAT_GT:

  1119       emitFloatCmp(above);

  1120       break;

  1122     case OP_FLOAT_GE:

  1123       emitFloatCmp(above_equal);

  1124       break;

  1126     case OP_FLOAT_LE:

  1127       emitFloatCmp(below_equal);

  1128       break;

  1130     case OP_FLOAT_LT:

  1131       emitFloatCmp(below);

  1132       break;

  1134     case OP_FLOAT_EQ:

  1135       emitFloatCmp(equal);

  1136       break;

  1138     case OP_FLOAT_NE:

  1139       emitFloatCmp(not_equal);

  1140       break;

  1142     case OP_FLOAT_NOT:

  1143     {

  1144       if (MacroAssemblerX86::Features().sse) {

  1145         __ xorps(xmm0, xmm0);

  1146         __ ucomiss(Operand(stk, 0), xmm0);

  1147       } else {

  1148         __ fld32(Operand(stk, 0));

  1149         __ fldz();

  1150         __ fucomip(st1);

  1151         __ fstp(st0);

  1152       }

  1154       // See emitFloatCmp() - this is a shorter version.

  1155       Label done;

  1156       __ movl(eax, 1);

  1157       __ j(parity, &done);

  1158       __ set(zero, r8_al);

  1159       __ bind(&done);

  1161       __ addl(stk, 4);

  1162       break;

  1163     }

  1165     case OP_STACK:

  1166     {

  1167       cell_t amount = readCell();

  1168       __ addl(stk, amount);

  1170      if (amount > 0) {

  1171        // Check if the stack went beyond the stack top - usually a compiler error.

  1172        __ cmpl(stk, intptr_t(plugin_->memory + plugin_->mem_size));

  1173        __ j(not_below, &error_stack_min_);

  1174      } else {

  1175        // Check if the stack is going to collide with the heap.

  1176        __ movl(tmp, Operand(hpAddr()));

  1177        __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1178        __ cmpl(stk, tmp);

  1179        __ j(below, &error_stack_low_);

  1180      }

  1181      break;

  1182     }

  1184     case OP_HEAP:

  1185     {

  1186       cell_t amount = readCell();

  1187       __ movl(alt, Operand(hpAddr()));

  1188       __ addl(Operand(hpAddr()), amount);

  1190       if (amount < 0) {

  1191         __ cmpl(Operand(hpAddr()), plugin_->data_size);

  1192         __ j(below, &error_heap_min_);

  1193       } else {

  1194         __ movl(tmp, Operand(hpAddr()));

  1195         __ lea(tmp, Operand(dat, ecx, NoScale, STACK_MARGIN));

  1196         __ cmpl(tmp, stk);

  1197         __ j(above, &error_heap_low_);

  1198       }

  1199       break;

  1200     }

  1202     case OP_JUMP:

  1203     {

  1204       Label *target = labelAt(readCell());

  1205       if (!target)

  1206         return false;

  1207       if (target->bound()) {

  1208         __ jmp32(target);

  1209         backward_jumps_.append(masm.pc());

  1210       } else {

  1211         __ jmp(target);

  1212       }

  1213       break;

  1214     }

  1216     case OP_JZER:

  1217     case OP_JNZ:

  1218     {

  1219       ConditionCode cc = (op == OP_JZER) ? zero : not_zero;

  1220       Label *target = labelAt(readCell());

  1221       if (!target)

  1222         return false;

  1223       __ testl(pri, pri);

  1224       if (target->bound()) {

  1225         __ j32(cc, target);

  1226         backward_jumps_.append(masm.pc());

  1227       } else {

  1228         __ j(cc, target);

  1229       }

  1230       break;

  1231     }

  1233     case OP_JEQ:

  1234     case OP_JNEQ:

  1235     case OP_JSLESS:

  1236     case OP_JSLEQ:

  1237     case OP_JSGRTR:

  1238     case OP_JSGEQ:

  1239     {

  1240       Label *target = labelAt(readCell());

  1241       if (!target)

  1242         return false;

  1243       ConditionCode cc = OpToCondition(op);

  1244       __ cmpl(pri, alt);

  1245       if (target->bound()) {

  1246         __ j32(cc, target);

  1247         backward_jumps_.append(masm.pc());

  1248       } else {

  1249         __ j(cc, target);

  1250       }

  1251       break;

  1252     }

  1254     case OP_TRACKER_PUSH_C:

  1255     {

  1256       cell_t amount = readCell();

  1258       __ push(pri);

  1259       __ push(alt);

  1261       __ push(amount * 4);

  1262       __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));

  1263       __ call(ExternalAddress((void *)PushTracker));

  1264       __ addl(esp, 8);

  1265       __ testl(eax, eax);

  1266       __ j(not_zero, &extern_error_);

  1268       __ pop(alt);

  1269       __ pop(pri);

  1270       break;

  1271     }

  1273     case OP_TRACKER_POP_SETHEAP:

  1274     {

  1275       // Save registers.

  1276       __ push(pri);

  1277       __ push(alt);

  1279       // Get the context pointer and call the sanity checker.

  1280       __ push(intptr_t(rt_));

  1281       __ call(ExternalAddress((void *)PopTrackerAndSetHeap));

  1282       __ addl(esp, 4);

  1283       __ testl(eax, eax);

  1284       __ j(not_zero, &extern_error_);

  1286       __ pop(alt);

  1287       __ pop(pri);

  1288       break;

  1289     }

  1291     case OP_BREAK:

  1292     {

  1293       cell_t cip = uintptr_t(cip_ - 1) - uintptr_t(plugin_->pcode);

  1294       __ movl(Operand(cipAddr()), cip);

  1295       break;

  1296     }

  1298     case OP_HALT:

  1299       __ align(16);

  1300       __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1301       __ movl(Operand(tmp, offsetof(sp_context_t, rval)), pri);

  1302       __ movl(pri, readCell());

  1303       __ jmp(&extern_error_);

  1304       break;

  1306     case OP_BOUNDS:

  1307     {

  1308       cell_t value = readCell();

  1309       __ cmpl(eax, value);

  1310       __ j(above, &error_bounds_);

  1311       break;

  1312     }

  1314     case OP_GENARRAY:

  1315     case OP_GENARRAY_Z:

  1316       emitGenArray(op == OP_GENARRAY_Z);

  1317       break;

  1319     case OP_CALL:

  1320       if (!emitCall())

  1321         return false;

  1322       break;

  1324     case OP_SYSREQ_C:

  1325     case OP_SYSREQ_N:

  1326       if (!emitNativeCall(op))

  1327         return false;

  1328       break;

  1330     case OP_SWITCH:

  1331       if (!emitSwitch())

  1332         return false;

  1333       break;

  1335     case OP_CASETBL:

  1336     {

  1337       size_t ncases = readCell();

  1339       // Two cells per case, and one extra cell for the default address.

  1340       cip_ += (ncases * 2) + 1;

  1341       break;

  1342     }

  1344     case OP_NOP:

  1345       break;

  1347     default:

  1348       error_ = SP_ERROR_INVALID_INSTRUCTION;

  1349       return false;

  1350   }

  1352   return true;

  1353 }

  1355 Label *

  1356 Compiler::labelAt(size_t offset)

  1357 {

  1358   if (offset % 4 != 0 ||

  1359       offset > plugin_->pcode_size ||

  1360       offset <= pcode_start_)

  1361   {

  1362     // If the jump target is misaligned, or out of pcode bounds, or is an

  1363     // address out of the function bounds, we abort. Unfortunately we can't

  1364     // test beyond the end of the function since we don't have a precursor

  1365     // pass (yet).

  1366     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1367     return NULL;

  1368   }

  1370   return &jump_map_[offset / sizeof(cell_t)];

  1371 }

  1373 void

  1374 Compiler::emitCheckAddress(Register reg)

  1375 {

  1376   // Check if we're in memory bounds.

  1377   __ cmpl(reg, plugin_->mem_size);

  1378   __ j(not_below, &error_memaccess_);

  1380   // Check if we're in the invalid region between hp and sp.

  1381   Label done;

  1382   __ cmpl(reg, Operand(hpAddr()));

  1383   __ j(below, &done);

  1384   __ lea(tmp, Operand(dat, reg, NoScale));

  1385   __ cmpl(tmp, stk);

  1386   __ j(below, &error_memaccess_);

  1387   __ bind(&done);

  1388 }

  1390 void

  1391 Compiler::emitGenArray(bool autozero)

  1392 {

  1393   cell_t val = readCell();

  1394   if (val == 1)

  1395   {

  1396     // flat array; we can generate this without indirection tables.

  1397     // Note that we can overwrite ALT because technically STACK should be destroying ALT

  1398     __ movl(alt, Operand(hpAddr()));

  1399     __ movl(tmp, Operand(stk, 0));

  1400     __ movl(Operand(stk, 0), alt);    // store base of the array into the stack.

  1401     __ lea(alt, Operand(alt, tmp, ScaleFour));

  1402     __ movl(Operand(hpAddr()), alt);

  1403     __ addl(alt, dat);

  1404     __ cmpl(alt, stk);

  1405     __ j(not_below, &error_heap_low_);

  1407     __ shll(tmp, 2);

  1408     __ push(tmp);

  1409     __ push(intptr_t(rt_->GetBaseContext()->GetCtx()));

  1410     __ call(ExternalAddress((void *)PushTracker));

  1411     __ addl(esp, 4);

  1412     __ pop(tmp);

  1413     __ shrl(tmp, 2);

  1414     __ testl(eax, eax);

  1415     __ j(not_zero, &extern_error_);

  1417     if (autozero) {

  1418       // Note - tmp is ecx and still intact.

  1419       __ push(eax);

  1420       __ push(edi);

  1421       __ xorl(eax, eax);

  1422       __ movl(edi, Operand(stk, 0));

  1423       __ addl(edi, dat);

  1424       __ cld();

  1425       __ rep_stosd();

  1426       __ pop(edi);

  1427       __ pop(eax);

  1428     }

  1429   } else {

  1430     __ push(pri);

  1432     // int GenerateArray(sp_plugin_t, vars[], uint32_t, cell_t *, int, unsigned *);

  1433     __ push(autozero ? 1 : 0);

  1434     __ push(stk);

  1435     __ push(val);

  1436     __ push(intptr_t(rt_));

  1437     __ call(ExternalAddress((void *)GenerateFullArray));

  1438     __ addl(esp, 4 * sizeof(void *));

  1440     // restore pri to tmp

  1441     __ pop(tmp);

  1443     __ testl(eax, eax);

  1444     __ j(not_zero, &extern_error_);

  1446     // Move tmp back to pri, remove pushed args.

  1447     __ movl(pri, tmp);

  1448     __ addl(stk, (val - 1) * 4);

  1449   }

  1450 }

  1452 bool

  1453 Compiler::emitCall()

  1454 {

  1455   cell_t offset = readCell();

  1457   // If this offset looks crappy, i.e. not aligned or out of bounds, we just

  1458   // abort.

  1459   if (offset % 4 != 0 || uint32_t(offset) >= plugin_->pcode_size) {

  1460     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1461     return false;

  1462   }

  1464   // eax = context

  1465   // ecx = rp

  1466   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1467   __ movl(ecx, Operand(eax, offsetof(sp_context_t, rp)));

  1469   // Check if the return stack is used up.

  1470   __ cmpl(ecx, SP_MAX_RETURN_STACK);

  1471   __ j(not_below, &error_stack_low_);

  1473   // Add to the return stack.

  1474   uintptr_t cip = uintptr_t(cip_ - 2) - uintptr_t(plugin_->pcode);

  1475   __ movl(Operand(eax, ecx, ScaleFour, offsetof(sp_context_t, rstk_cips)), cip);

  1477   // Increment the return stack pointer.

  1478   __ addl(Operand(eax, offsetof(sp_context_t, rp)), 1);

  1480   // Store the CIP of the function we're about to call.

  1481   __ movl(Operand(cipAddr()), offset);

  1483   JitFunction *fun = rt_->GetJittedFunctionByOffset(offset);

  1484   if (!fun) {

  1485     // Need to emit a delayed thunk.

  1486     CallThunk *thunk = new CallThunk(offset);

  1487     __ call(&thunk->call);

  1488     if (!thunks_.append(thunk))

  1489       return false;

  1490   } else {

  1491     // Function is already emitted, we can do a direct call.

  1492     __ call(ExternalAddress(fun->GetEntryAddress()));

  1493   }

  1495   // Restore the last cip.

  1496   __ movl(Operand(cipAddr()), cip);

  1498   // Mark us as leaving the last frame.

  1499   __ movl(tmp, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1500   __ subl(Operand(tmp, offsetof(sp_context_t, rp)), 1);

  1501   return true;

  1502 }

  1504 void

  1505 Compiler::emitCallThunks()

  1506 {

  1507   for (size_t i = 0; i < thunks_.length(); i++) {

  1508     CallThunk *thunk = thunks_[i];

  1510     Label error;

  1511     __ bind(&thunk->call);

  1512     // Huge hack - get the return address, since that is the call that we

  1513     // need to patch.

  1514     __ movl(eax, Operand(esp, 0));

  1516     // We need to push 4 arguments, and one of them will need an extra word

  1517     // on the stack. Allocate a big block so we're aligned, subtracting

  1518     // 4 because we got here via a call.

  1519     static const size_t kStackNeeded = 5 * sizeof(void *);

  1520     static const size_t kStackReserve = ke::Align(kStackNeeded, 16) - sizeof(void *);

  1521     __ subl(esp, kStackReserve);

  1523     // Set arguments.

  1524     __ movl(Operand(esp, 3 * sizeof(void *)), eax);

  1525     __ lea(edx, Operand(esp, 4 * sizeof(void *)));

  1526     __ movl(Operand(esp, 2 * sizeof(void *)), edx);

  1527     __ movl(Operand(esp, 1 * sizeof(void *)), intptr_t(thunk->pcode_offset));

  1528     __ movl(Operand(esp, 0 * sizeof(void *)), intptr_t(rt_));

  1530     __ call(ExternalAddress((void *)CompileFromThunk));

  1531     __ movl(edx, Operand(esp, 4 * sizeof(void *)));

  1532     __ addl(esp, kStackReserve);

  1533     __ testl(eax, eax);

  1534     __ j(not_zero, &error);

  1535     __ jmp(edx);

  1537     __ bind(&error);

  1538     __ movl(Operand(cipAddr()), thunk->pcode_offset);

  1539     __ jmp(g_Jit.GetUniversalReturn());

  1540   }

  1541 }

  1543 cell_t

  1544 Compiler::readCell()

  1545 {

  1546   if (cip_ >= code_end_) {

  1547     error_= SP_ERROR_INVALID_INSTRUCTION;

  1548     return 0;

  1549   }

  1550   return *cip_++;

  1551 }

  1553 bool

  1554 Compiler::emitNativeCall(OPCODE op)

  1555 {

  1556   uint32_t native_index = readCell();

  1558   if (native_index >= plugin_->num_natives) {

  1559     error_ = SP_ERROR_INSTRUCTION_PARAM;

  1560     return false;

  1561   }

  1563   uint32_t num_params;

  1564   if (op == OP_SYSREQ_N) {

  1565     num_params = readCell();

  1567     // See if we can get a replacement opcode. If we can, then recursively

  1568     // call emitOp() to generate it. Note: it's important that we do this

  1569     // before generating any code for the SYSREQ.N.

  1570     unsigned replacement = rt_->GetNativeReplacement(native_index);

  1571     if (replacement != OP_NOP)

  1572       return emitOp((OPCODE)replacement);

  1574     // Store the number of parameters.

  1575     __ movl(Operand(stk, -4), num_params);

  1576     __ subl(stk, 4);

  1577   }

  1579   // Save registers.

  1580   __ push(edx);

  1582   // Push the last parameter for the C++ function.

  1583   __ push(stk);

  1585   // Relocate our absolute stk to be dat-relative, and update the context's

  1586   // view.

  1587   __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1588   __ subl(stk, dat);

  1589   __ movl(Operand(eax, offsetof(sp_context_t, sp)), stk);

  1591   __ movl(Operand(eax, offsetof(sp_context_t, n_idx)), native_index);

  1593   sp_native_t *native = rt_->GetNativeByIndex(native_index);

  1594   if ((native->status != SP_NATIVE_BOUND) ||

  1595       (native->flags & (SP_NTVFLAG_OPTIONAL | SP_NTVFLAG_EPHEMERAL)))

  1596   {

  1597     // The native is either unbound, or it could become unbound in the

  1598     // future. Invoke the slower native callback.

  1599     __ push(native_index);

  1600     __ push(eax);

  1601     __ call(ExternalAddress((void *)NativeCallback));

  1602   } else {

  1603     // The native is bound so we have a few more guarantees.

  1604     __ push(intptr_t(native->pfn));

  1605     __ push(eax);

  1606     __ call(ExternalAddress((void *)BoundNativeCallback));

  1607   }

  1609   // Check for errors.

  1610   __ movl(ecx, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1611   __ movl(ecx, Operand(ecx, offsetof(sp_context_t, n_err)));

  1612   __ testl(ecx, ecx);

  1613   __ j(not_zero, &extern_error_);

  1615   // Restore local state.

  1616   __ addl(stk, dat);

  1617   __ addl(esp, 12);

  1618   __ pop(edx);

  1620   if (op == OP_SYSREQ_N) {

  1621     // Pop the stack. Do not check the margins.

  1622     __ addl(stk, (num_params + 1) * sizeof(cell_t));

  1623   }

  1624   return true;

  1625 }

  1627 bool

  1628 Compiler::emitSwitch()

  1629 {

  1630   cell_t offset = readCell();

  1631   if (!labelAt(offset))

  1632     return false;

  1634   cell_t *tbl = (cell_t *)((char *)plugin_->pcode + offset + sizeof(cell_t));

  1636   struct Entry {

  1637     cell_t val;

  1638     cell_t offset;

  1639   };

  1641   size_t ncases = *tbl++;

  1643   Label *defaultCase = labelAt(*tbl);

  1644   if (!defaultCase)

  1645     return false;

  1647   // Degenerate - 0 cases.

  1648   if (!ncases) {

  1649     __ jmp(defaultCase);

  1650     return true;

  1651   }

  1653   Entry *cases = (Entry *)(tbl + 1);

  1655   // Degenerate - 1 case.

  1656   if (ncases == 1) {

  1657     Label *maybe = labelAt(cases[0].offset);

  1658     if (!maybe)

  1659       return false;

  1660     __ cmpl(pri, cases[0].val);

  1661     __ j(equal, maybe);

  1662     __ jmp(defaultCase);

  1663     return true;

  1664   }

  1666   // We have two or more cases, so let's generate a full switch. Decide

  1667   // whether we'll make an if chain, or a jump table, based on whether

  1668   // the numbers are strictly sequential.

  1669   bool sequential = true;

  1670   {

  1671     cell_t first = cases[0].val;

  1672     cell_t last = first;

  1673     for (size_t i = 1; i < ncases; i++) {

  1674       if (cases[i].val != ++last) {

  1675         sequential = false;

  1676         break;

  1677       }

  1678     }

  1679   }

  1681   // First check whether the bounds are correct: if (a < LOW || a > HIGH);

  1682   // this check is valid whether or not we emit a sequential-optimized switch.

  1683   cell_t low = cases[0].val;

  1684   if (low != 0) {

  1685     // negate it so we'll get a lower bound of 0.

  1686     low = -low;

  1687     __ lea(tmp, Operand(pri, low));

  1688   } else {

  1689     __ movl(tmp, pri);

  1690   }

  1692   cell_t high = abs(cases[0].val - cases[ncases - 1].val);

  1693   __ cmpl(tmp, high);

  1694   __ j(above, defaultCase);

  1696   if (sequential) {

  1697     // Optimized table version. The tomfoolery below is because we only have

  1698     // one free register... it seems unlikely pri or alt will be used given

  1699     // that we're at the end of a control-flow point, but we'll play it safe.

  1700     DataLabel table;

  1701     __ push(eax);

  1702     __ movl(eax, &table);

  1703     __ movl(ecx, Operand(eax, ecx, ScaleFour));

  1704     __ pop(eax);

  1705     __ jmp(ecx);

  1707     __ bind(&table);

  1708     for (size_t i = 0; i < ncases; i++) {

  1709       Label *label = labelAt(cases[i].offset);

  1710       if (!label)

  1711         return false;

  1712       __ emit_absolute_address(label);

  1713     }

  1714   } else {

  1715     // Slower version. Go through each case and generate a check.

  1716     for (size_t i = 0; i < ncases; i++) {

  1717       Label *label = labelAt(cases[i].offset);

  1718       if (!label)

  1719         return false;

  1720       __ cmpl(pri, cases[i].val);

  1721       __ j(equal, label);

  1722     }

  1723     __ jmp(defaultCase);

  1724   }

  1725   return true;

  1726 }

  1728 void

  1729 Compiler::emitErrorPath(Label *dest, int code)

  1730 {

  1731   if (dest->used()) {

  1732     __ bind(dest);

  1733     __ movl(eax, code);

  1734     __ jmp(g_Jit.GetUniversalReturn());

  1735   }

  1736 }

  1738 void

  1739 Compiler::emitFloatCmp(ConditionCode cc)

  1740 {

  1741   unsigned lhs = 4;

  1742   unsigned rhs = 0;

  1743   if (cc == below || cc == below_equal) {

  1744     // NaN results in ZF=1 PF=1 CF=1

  1745     //

  1746     // ja/jae check for ZF,CF=0 and CF=0. If we make all relational compares

  1747     // look like ja/jae, we'll guarantee all NaN comparisons will fail (which

  1748     // would not be true for jb/jbe, unless we checked with jp).

  1749     if (cc == below)

  1750       cc = above;

  1751     else

  1752       cc = above_equal;

  1753     rhs = 4;

  1754     lhs = 0;

  1755   }

  1757   if (MacroAssemblerX86::Features().sse) {

  1758     __ movss(xmm0, Operand(stk, rhs));

  1759     __ ucomiss(Operand(stk, lhs), xmm0);

  1760   } else {

  1761     __ fld32(Operand(stk, rhs));

  1762     __ fld32(Operand(stk, lhs));

  1763     __ fucomip(st1);

  1764     __ fstp(st0);

  1765   }

  1767   // An equal or not-equal needs special handling for the parity bit.

  1768   if (cc == equal || cc == not_equal) {

  1769     // If NaN, PF=1, ZF=1, and E/Z tests ZF=1.

  1770     //

  1771     // If NaN, PF=1, ZF=1 and NE/NZ tests Z=0. But, we want any != with NaNs

  1772     // to return true, including NaN != NaN.

  1773     //

  1774     // To make checks simpler, we set |eax| to the expected value of a NaN

  1775     // beforehand. This also clears the top bits of |eax| for setcc.

  1776     Label done;

  1777     __ movl(eax, (cc == equal) ? 0 : 1);

  1778     __ j(parity, &done);

  1779     __ set(cc, r8_al);

  1780     __ bind(&done);

  1781   } else {

  1782     __ movl(eax, 0);

  1783     __ set(cc, r8_al);

  1784   }

  1785   __ addl(stk, 8);

  1786 }

  1788 void

  1789 Compiler::emitErrorPaths()

  1790 {

  1791   emitErrorPath(&error_divide_by_zero_, SP_ERROR_DIVIDE_BY_ZERO);

  1792   emitErrorPath(&error_stack_low_, SP_ERROR_STACKLOW);

  1793   emitErrorPath(&error_stack_min_, SP_ERROR_STACKMIN);

  1794   emitErrorPath(&error_bounds_, SP_ERROR_ARRAY_BOUNDS);

  1795   emitErrorPath(&error_memaccess_, SP_ERROR_MEMACCESS);

  1796   emitErrorPath(&error_heap_low_, SP_ERROR_HEAPLOW);

  1797   emitErrorPath(&error_heap_min_, SP_ERROR_HEAPMIN);

  1798   emitErrorPath(&error_integer_overflow_, SP_ERROR_INTEGER_OVERFLOW);

  1800   if (extern_error_.used()) {

  1801     __ bind(&extern_error_);

  1802     __ movl(eax, intptr_t(rt_->GetBaseContext()->GetCtx()));

  1803     __ movl(eax, Operand(eax, offsetof(sp_context_t, n_err)));

  1804     __ jmp(g_Jit.GetUniversalReturn());

  1805   }

  1806 }

  1808 typedef int (*JIT_EXECUTE)(sp_context_t *ctx, uint8_t *memory, void *code);

  1810 static void *

  1811 GenerateEntry(void **retp, void **timeoutp)

  1812 {

  1813   AssemblerX86 masm;

  1815   __ push(ebp);

  1816   __ movl(ebp, esp);

  1818   __ push(esi);   // ebp - 4

  1819   __ push(edi);   // ebp - 8

  1820   __ push(ebx);   // ebp - 12

  1821   __ push(esp);   // ebp - 16

  1823   __ movl(ebx, Operand(ebp, 8 + 4 * 0));

  1824   __ movl(eax, Operand(ebp, 8 + 4 * 1));

  1825   __ movl(ecx, Operand(ebp, 8 + 4 * 2));

  1827   // Set up run-time registers.

  1828   __ movl(edi, Operand(ebx, offsetof(sp_context_t, sp)));

  1829   __ addl(edi, eax);

  1830   __ movl(esi, eax);

  1831   __ movl(ebx, edi);

  1833   // Align the stack.

  1834   __ andl(esp, 0xfffffff0);

  1836   // Call into plugin (align the stack first).

  1837   __ call(ecx);

  1839   // Get input context, store rval.

  1840   __ movl(ecx, Operand(ebp, 8 + 4 * 0));

  1841   __ movl(Operand(ecx, offsetof(sp_context_t, rval)), pri);

  1843   // Set no error.

  1844   __ movl(eax, SP_ERROR_NONE);

  1846   // Store latest stk. If we have an error code, we'll jump directly to here,

  1847   // so eax will already be set.

  1848   Label ret;

  1849   __ bind(&ret);

  1850   __ subl(stk, dat);

  1851   __ movl(Operand(ecx, offsetof(sp_context_t, sp)), stk);

  1853   // Restore stack.

  1854   __ movl(esp, Operand(ebp, -16));

  1856   // Restore registers and gtfo.

  1857   __ pop(ebx);

  1858   __ pop(edi);

  1859   __ pop(esi);

  1860   __ pop(ebp);

  1861   __ ret();

  1863   // The universal emergency return will jump to here.

  1864   Label error;

  1865   __ bind(&error);

  1866   __ movl(ecx, Operand(ebp, 8 + 4 * 0)); // ret-path expects ecx = ctx

  1867   __ jmp(&ret);

  1869   Label timeout;

  1870   __ bind(&timeout);

  1871   __ movl(eax, SP_ERROR_TIMEOUT);

  1872   __ jmp(&error);

  1874   void *code = LinkCode(masm);

  1875   if (!code)

  1876     return NULL;

  1878   *retp = reinterpret_cast<uint8_t *>(code) + error.offset();

  1879   *timeoutp = reinterpret_cast<uint8_t *>(code) + timeout.offset();

  1880   return code;

  1881 }

  1883 ICompilation *JITX86::ApplyOptions(ICompilation *_IN, ICompilation *_OUT)

  1884 {

  1885   if (_IN == NULL)

  1886     return _OUT;

  1888   CompData *_in = (CompData * )_IN;

  1889   CompData *_out = (CompData * )_OUT;

  1891   _in->inline_level = _out->inline_level;

  1892   _in->profile = _out->profile;

  1894   _out->Abort();

  1895   return _in;

  1896 }

  1898 JITX86::JITX86()

  1899 {

  1900   m_pJitEntry = NULL;

  1901 }

  1903 bool

  1904 JITX86::InitializeJIT()

  1905 {

  1906   g_pCodeCache = KE_CreateCodeCache();

  1908   m_pJitEntry = GenerateEntry(&m_pJitReturn, &m_pJitTimeout);

  1909   if (!m_pJitEntry)

  1910     return false;

  1912   MacroAssemblerX86 masm;

  1913   MacroAssemblerX86::GenerateFeatureDetection(masm);

  1914   void *code = LinkCode(masm);

  1915   if (!code)

  1916     return false;

  1917   MacroAssemblerX86::RunFeatureDetection(code);

  1918   KE_FreeCode(g_pCodeCache, code);

  1920   return true;

  1921 }

  1923 void

  1924 JITX86::ShutdownJIT()

  1925 {

  1926   KE_DestroyCodeCache(g_pCodeCache);

  1927 }

  1929 JitFunction *

  1930 JITX86::CompileFunction(BaseRuntime *prt, cell_t pcode_offs, int *err)

  1931 {

  1932   Compiler cc(prt, pcode_offs);

  1933   JitFunction *fun = cc.emit(err);

  1934   if (!fun)

  1935     return NULL;

  1937   // Grab the lock before linking code in, since the watchdog timer will look

  1938   // at this list on another thread.

  1939   ke::AutoLock lock(g_Jit.Mutex());

  1941   prt->AddJittedFunction(fun);

  1942   return fun;

  1943 }

  1945 void

  1946 JITX86::SetupContextVars(BaseRuntime *runtime, BaseContext *pCtx, sp_context_t *ctx)

  1947 {

  1948   ctx->tracker = new tracker_t;

  1949   ctx->tracker->pBase = (ucell_t *)malloc(1024);

  1950   ctx->tracker->pCur = ctx->tracker->pBase;

  1951   ctx->tracker->size = 1024 / sizeof(cell_t);

  1952   ctx->basecx = pCtx;

  1953   ctx->vm[JITVARS_PROFILER] = g_engine2.GetProfiler();

  1954   ctx->plugin = const_cast<sp_plugin_t *>(runtime->plugin());

  1955 }

  1957 SPVM_NATIVE_FUNC

  1958 JITX86::CreateFakeNative(SPVM_FAKENATIVE_FUNC callback, void *pData)

  1959 {

  1960   AssemblerX86 masm;

  1962   __ push(ebx);

  1963   __ push(edi);

  1964   __ push(esi);

  1965   __ movl(edi, Operand(esp, 16)); // store ctx

  1966   __ movl(esi, Operand(esp, 20)); // store params

  1967   __ movl(ebx, esp);

  1968   __ andl(esp, 0xfffffff0);

  1969   __ subl(esp, 4);

  1971   __ push(intptr_t(pData));

  1972   __ push(esi);

  1973   __ push(edi);

  1974   __ call(ExternalAddress((void *)callback));

  1975   __ movl(esp, ebx);

  1976   __ pop(esi);

  1977   __ pop(edi);

  1978   __ pop(ebx);

  1979   __ ret();

  1981   return (SPVM_NATIVE_FUNC)LinkCode(masm);

  1982 }

  1984 void

  1985 JITX86::DestroyFakeNative(SPVM_NATIVE_FUNC func)

  1986 {

  1987   KE_FreeCode(g_pCodeCache, (void *)func);

  1988 }

  1990 ICompilation *

  1991 JITX86::StartCompilation()

  1992 {

  1993   return new CompData;

  1994 }

  1996 ICompilation *

  1997 JITX86::StartCompilation(BaseRuntime *runtime)

  1998 {

  1999   return new CompData;

  2000 }

  2002 void

  2003 CompData::Abort()

  2004 {

  2005   delete this;

  2006 }

  2008 void

  2009 JITX86::FreeContextVars(sp_context_t *ctx)

  2010 {

  2011   free(ctx->tracker->pBase);

  2012   delete ctx->tracker;

  2013 }

  2015 bool

  2016 CompData::SetOption(const char *key, const char *val)

  2017 {

  2018   if (strcmp(key, SP_JITCONF_DEBUG) == 0)

  2019     return true;

  2020   if (strcmp(key, SP_JITCONF_PROFILE) == 0) {

  2021     profile = atoi(val);

  2023     /** Callbacks must be profiled to profile functions! */

  2024     if ((profile & SP_PROF_FUNCTIONS) == SP_PROF_FUNCTIONS)

  2025       profile |= SP_PROF_CALLBACKS;

  2027     return true;

  2028   }

  2030   return false;

  2031 }

  2033 int

  2034 JITX86::InvokeFunction(BaseRuntime *runtime, JitFunction *fn, cell_t *result)

  2035 {

  2036   sp_context_t *ctx = runtime->GetBaseContext()->GetCtx();

  2038   // Note that cip, hp, sp are saved and restored by Execute2().

  2039   ctx->cip = fn->GetPCodeAddress();

  2041   JIT_EXECUTE pfn = (JIT_EXECUTE)m_pJitEntry;

  2043   if (level_++ == 0)

  2044     frame_id_++;

  2045   int err = pfn(ctx, runtime->plugin()->memory, fn->GetEntryAddress());

  2046   level_--;

  2048   *result = ctx->rval;

  2049   return err;

  2050 }

  2052 void *

  2053 JITX86::AllocCode(size_t size)

  2054 {

  2055   return Knight::KE_AllocCode(g_pCodeCache, size);

  2056 }

  2058 void

  2059 JITX86::FreeCode(void *code)

  2060 {

  2061   KE_FreeCode(g_pCodeCache, code);

  2062 }

  2064 void

  2065 JITX86::RegisterRuntime(BaseRuntime *rt)

  2066 {

  2067   mutex_.AssertCurrentThreadOwns();

  2068   runtimes_.append(rt);

  2069 }

  2071 void

  2072 JITX86::DeregisterRuntime(BaseRuntime *rt)

  2073 {

  2074   mutex_.AssertCurrentThreadOwns();

  2075   runtimes_.remove(rt);

  2076 }

  2078 void

  2079 JITX86::PatchAllJumpsForTimeout()

  2080 {

  2081   mutex_.AssertCurrentThreadOwns();

  2082   for (ke::InlineList<BaseRuntime>::iterator iter = runtimes_.begin(); iter != runtimes_.end(); iter++) {

  2083     BaseRuntime *rt = *iter;

  2084     for (size_t i = 0; i < rt->NumJitFunctions(); i++) {

  2085       JitFunction *fun = rt->GetJitFunction(i);

  2086       uint8_t *base = reinterpret_cast<uint8_t *>(fun->GetEntryAddress());

  2088       for (size_t j = 0; j < fun->NumLoopEdges(); j++) {

  2089         const LoopEdge &e = fun->GetLoopEdge(j);

  2090         int32_t diff = intptr_t(m_pJitTimeout) - intptr_t(base + e.offset);

  2091         *reinterpret_cast<int32_t *>(base + e.offset - 4) = diff;

  2092       }

  2093     }

  2094   }

  2095 }

  2097 void

  2098 JITX86::UnpatchAllJumpsFromTimeout()

  2099 {

  2100   mutex_.AssertCurrentThreadOwns();

  2101   for (ke::InlineList<BaseRuntime>::iterator iter = runtimes_.begin(); iter != runtimes_.end(); iter++) {

  2102     BaseRuntime *rt = *iter;

  2103     for (size_t i = 0; i < rt->NumJitFunctions(); i++) {

  2104       JitFunction *fun = rt->GetJitFunction(i);

  2105       uint8_t *base = reinterpret_cast<uint8_t *>(fun->GetEntryAddress());

  2107       for (size_t j = 0; j < fun->NumLoopEdges(); j++) {

  2108         const LoopEdge &e = fun->GetLoopEdge(j);

  2109         *reinterpret_cast<int32_t *>(base + e.offset - 4) = e.disp32;

  2110       }

  2111     }

  2112   }

  2113 }

author	Kyle Sanderson <kyle.leet@gmail.com>
	Sun May 11 14:36:32 2014 -0700 (2014-05-11)
changeset 4358	52aa5fab9866
parent 4327	60e4042f2589
permissions	-rw-r--r--