Move tracker related opcodes entirely to C++.
Move tracker related opcodes entirely to C++.
The tracker related opcodes:
GENARRAY
GENARRAY_Z
TRACKER_POP_SETHEAP
TRACKER_PUSH_C
All contain some vastly overcomplicated assembly containing logic that could
be implemented much easier in C++. If it were a performance concern, these
opcodes would be entirely in C++, but most of them call out to one or more
routines to do additional work. This patch just moves most of the logic out
to C++ to reduce complexity and fix reported bugs.
1.1 --- a/sourcepawn/jit/x86/jit_x86.cpp Sat Aug 10 12:22:09 2013 -0400
1.2 +++ b/sourcepawn/jit/x86/jit_x86.cpp Sat Aug 10 21:23:02 2013 -0700
1.3 @@ -143,7 +143,7 @@
1.4 return size;
1.5 }
1.6
1.7 -static void
1.8 +static cell_t
1.9 GenerateArrayIndirectionVectors(cell_t *arraybase, cell_t dims[], cell_t _dimcount, bool autozero)
1.10 {
1.11 array_creation_t ar;
1.12 @@ -161,28 +161,37 @@
1.13 ar.data_offs = &data_offs;
1.14
1.15 data_offs = calc_indirection(&ar, 0);
1.16 -
1.17 GenerateInnerArrayIndirectionVectors(&ar, 0, 0);
1.18 + return data_offs;
1.19 }
1.20
1.21 static int
1.22 -JIT_VerifyLowBoundTracker(sp_context_t *ctx)
1.23 +PopTrackerAndSetHeap(sp_plugin_t *plugin, InfoVars &vars)
1.24 {
1.25 - tracker_t *trk = (tracker_t *)(ctx->vm[JITVARS_TRACKER]);
1.26 - if (trk->pCur <= trk->pBase)
1.27 - return SP_ERROR_TRACKER_BOUNDS;
1.28 - return SP_ERROR_NONE;
1.29 + tracker_t *trk = (tracker_t *)(vars.ctx->vm[JITVARS_TRACKER]);
1.30 + assert(trk->pCur > trk->pBase);
1.31 +
1.32 + trk->pCur--;
1.33 + if (trk->pCur < trk->pBase)
1.34 + return SP_ERROR_TRACKER_BOUNDS;
1.35 +
1.36 + ucell_t amt = *trk->pCur;
1.37 + if (amt > (vars.hp - plugin->data_size))
1.38 + return SP_ERROR_HEAPMIN;
1.39 +
1.40 + vars.hp -= amt;
1.41 + return SP_ERROR_NONE;
1.42 }
1.43
1.44 static int
1.45 -JIT_VerifyOrAllocateTracker(sp_context_t *ctx)
1.46 +PushTracker(sp_context_t *ctx, size_t amount)
1.47 {
1.48 tracker_t *trk = (tracker_t *)(ctx->vm[JITVARS_TRACKER]);
1.49
1.50 if ((size_t)(trk->pCur - trk->pBase) >= trk->size)
1.51 return SP_ERROR_TRACKER_BOUNDS;
1.52
1.53 - if (trk->pCur+1 - (trk->pBase + trk->size) == 0) {
1.54 + if (trk->pCur + 1 - (trk->pBase + trk->size) == 0) {
1.55 size_t disp = trk->size - 1;
1.56 trk->size *= 2;
1.57 trk->pBase = (ucell_t *)realloc(trk->pBase, trk->size * sizeof(cell_t));
1.58 @@ -193,6 +202,55 @@
1.59 trk->pCur = trk->pBase + disp;
1.60 }
1.61
1.62 + *trk->pCur++ = amount;
1.63 + return SP_ERROR_NONE;
1.64 +}
1.65 +
1.66 +static int
1.67 +GenerateArray(sp_plugin_t *plugin, InfoVars &vars, uint32_t argc, cell_t *argv, int autozero)
1.68 +{
1.69 + // Calculate how many cells are needed.
1.70 + if (argv[0] <= 0)
1.71 + return SP_ERROR_ARRAY_TOO_BIG;
1.72 +
1.73 + uint32_t dim = 1; // second to last dimension
1.74 + uint32_t cells = argv[0];
1.75 +
1.76 + for (uint32_t dim = 1; dim < argc; dim++) {
1.77 + cell_t dimsize = argv[dim];
1.78 + if (dimsize <= 0)
1.79 + return SP_ERROR_ARRAY_TOO_BIG;
1.80 + if (!ke::IsUint32MultiplySafe(cells, dimsize))
1.81 + return SP_ERROR_ARRAY_TOO_BIG;
1.82 + cells *= uint32_t(dimsize);
1.83 + if (!ke::IsUint32AddSafe(cells, dimsize))
1.84 + return SP_ERROR_ARRAY_TOO_BIG;
1.85 + cells += uint32_t(dimsize);
1.86 + }
1.87 +
1.88 + if (!ke::IsUint32MultiplySafe(cells, 4))
1.89 + return SP_ERROR_ARRAY_TOO_BIG;
1.90 +
1.91 + uint32_t bytes = cells * 4;
1.92 + if (!ke::IsUint32AddSafe(vars.hp, bytes))
1.93 + return SP_ERROR_ARRAY_TOO_BIG;
1.94 +
1.95 + uint32_t new_hp = vars.hp + bytes;
1.96 + cell_t *dat_hp = reinterpret_cast<cell_t *>(plugin->memory + new_hp);
1.97 +
1.98 + // argv, coincidentally, is STK.
1.99 + if (dat_hp >= argv - STACK_MARGIN)
1.100 + return SP_ERROR_HEAPLOW;
1.101 +
1.102 + if (int err = PushTracker(vars.ctx, bytes))
1.103 + return err;
1.104 +
1.105 + cell_t *base = reinterpret_cast<cell_t *>(plugin->memory + vars.hp);
1.106 + cell_t offs = GenerateArrayIndirectionVectors(base, argv, argc, autozero);
1.107 + assert(size_t(offs) == cells);
1.108 +
1.109 + argv[argc - 1] = vars.hp;
1.110 + vars.hp = new_hp;
1.111 return SP_ERROR_NONE;
1.112 }
1.113
1.114 @@ -1199,23 +1257,16 @@
1.115 {
1.116 cell_t amount = readCell();
1.117
1.118 - // Save registers.
1.119 __ push(pri);
1.120 __ push(alt);
1.121
1.122 - // call JIT_VerifyorAllocateTracker
1.123 __ movl(eax, Operand(info, AMX_INFO_CONTEXT));
1.124 + __ push(amount * 4);
1.125 __ push(eax);
1.126 - __ call(ExternalAddress((void *)JIT_VerifyOrAllocateTracker));
1.127 + __ call(ExternalAddress((void *)PushTracker));
1.128 + __ addl(esp, 8);
1.129 __ testl(eax, eax);
1.130 __ j(not_zero, &extern_error_);
1.131 - __ pop(eax);
1.132 -
1.133 - // Push the value onto the stack and increment pCur.
1.134 - __ movl(edx, Operand(eax, offsetof(sp_context_t, vm[JITVARS_TRACKER])));
1.135 - __ movl(ecx, Operand(edx, offsetof(tracker_t, pCur)));
1.136 - __ addl(Operand(edx, offsetof(tracker_t, pCur)), 4);
1.137 - __ movl(Operand(ecx, 0), amount * 4);
1.138
1.139 __ pop(alt);
1.140 __ pop(pri);
1.141 @@ -1229,24 +1280,12 @@
1.142 __ push(alt);
1.143
1.144 // Get the context pointer and call the sanity checker.
1.145 - __ movl(eax, Operand(info, AMX_INFO_CONTEXT));
1.146 - __ push(eax);
1.147 - __ call(ExternalAddress((void *)JIT_VerifyLowBoundTracker));
1.148 + __ push(info);
1.149 + __ push(intptr_t(plugin_));
1.150 + __ call(ExternalAddress((void *)PopTrackerAndSetHeap));
1.151 + __ addl(esp, 8);
1.152 __ testl(eax, eax);
1.153 __ j(not_zero, &extern_error_);
1.154 - __ pop(eax);
1.155 -
1.156 - // Pop the value from the tracker stack and decrease the heap by it.
1.157 - __ movl(edx, Operand(eax, offsetof(sp_context_t, vm[JITVARS_TRACKER])));
1.158 - __ movl(ecx, Operand(edx, offsetof(tracker_t, pCur)));
1.159 - __ subl(ecx, 4);
1.160 - __ movl(Operand(edx, offsetof(tracker_t, pCur)), eax);
1.161 - __ movl(ecx, Operand(ecx, 0));
1.162 - __ subl(Operand(info, AMX_INFO_HEAP), ecx);
1.163 -
1.164 - // Check that we didn't underflow the heap.
1.165 - __ cmpl(Operand(info, AMX_INFO_HEAP), plugin_->data_size);
1.166 - __ j(below, &error_heap_min_);
1.167
1.168 __ pop(alt);
1.169 __ pop(pri);
1.170 @@ -1335,40 +1374,6 @@
1.171 return &jump_map_[offset / sizeof(cell_t)];
1.172 }
1.173
1.174 -static void
1.175 -EmitTrackerPushReg(AssemblerX86 &masm, Register reg, Label *error)
1.176 -{
1.177 - __ push(eax);
1.178 - if (reg == ecx)
1.179 - __ push(ecx);
1.180 - __ push(edi);
1.181 - __ movl(edi, reg);
1.182 - __ shll(edi, 2); // Count in bytes, not cells.
1.183 -
1.184 - // Get the context pointer, push it and call the check.
1.185 - __ movl(eax, Operand(info, AMX_INFO_CONTEXT));
1.186 - __ push(eax);
1.187 - __ call(ExternalAddress((void *)JIT_VerifyOrAllocateTracker));
1.188 -
1.189 - // Check for errors.
1.190 - __ testl(eax, eax);
1.191 - __ j(not_zero, error);
1.192 -
1.193 - __ pop(eax);
1.194 -
1.195 - // Push the register into the stack and increment pCur.
1.196 - __ movl(edx, Operand(eax, offsetof(sp_context_t, vm[JITVARS_TRACKER])));
1.197 - __ movl(eax, Operand(edx, offsetof(tracker_t, pCur)));
1.198 - __ addl(Operand(edx, offsetof(tracker_t, pCur)), 4);
1.199 - __ movl(Operand(eax, 0), edi);
1.200 -
1.201 - // Restore pri, alt, stk.
1.202 - __ pop(edi);
1.203 - if (reg == ecx)
1.204 - __ pop(ecx);
1.205 - __ pop(eax);
1.206 -}
1.207 -
1.208 void
1.209 Compiler::emitCheckAddress(Register reg)
1.210 {
1.211 @@ -1403,9 +1408,18 @@
1.212 __ cmpl(alt, stk);
1.213 __ j(not_below, &error_heap_low_);
1.214
1.215 - EmitTrackerPushReg(masm, ecx, &extern_error_);
1.216 + __ shll(tmp, 2);
1.217 + __ push(tmp);
1.218 + __ push(Operand(info, AMX_INFO_CONTEXT));
1.219 + __ call(ExternalAddress((void *)PushTracker));
1.220 + __ addl(esp, 4);
1.221 + __ pop(tmp);
1.222 + __ shrl(tmp, 2);
1.223 + __ testl(eax, eax);
1.224 + __ j(not_zero, &extern_error_);
1.225
1.226 if (autozero) {
1.227 + // Note - tmp is ecx and still intact.
1.228 __ push(eax);
1.229 __ push(edi);
1.230 __ xorl(eax, eax);
1.231 @@ -1417,9 +1431,26 @@
1.232 __ pop(eax);
1.233 }
1.234 } else {
1.235 - __ movl(tmp, val);
1.236 - __ movl(edx, autozero ? 1 : 0);
1.237 - __ call(g_Jit.GetGenArrayIntrinsic());
1.238 + __ push(pri);
1.239 +
1.240 + // int GenerateArray(sp_plugin_t, vars[], uint32_t, cell_t *, int, unsigned *);
1.241 + __ push(autozero ? 1 : 0);
1.242 + __ push(stk);
1.243 + __ push(val);
1.244 + __ push(info);
1.245 + __ push(intptr_t(plugin_));
1.246 + __ call(ExternalAddress((void *)GenerateArray));
1.247 + __ addl(esp, 5 * sizeof(void *));
1.248 +
1.249 + // restore pri to tmp
1.250 + __ pop(tmp);
1.251 +
1.252 + __ testl(eax, eax);
1.253 + __ j(not_zero, &extern_error_);
1.254 +
1.255 + // Move tmp back to pri, remove pushed args.
1.256 + __ movl(pri, tmp);
1.257 + __ addl(stk, (val - 1) * 4);
1.258 }
1.259 }
1.260
1.261 @@ -1728,90 +1759,6 @@
1.262 }
1.263 }
1.264
1.265 -// Input dimension count is in ecx.
1.266 -static void *
1.267 -GenerateGenArrayIntrinsic(void *ret)
1.268 -{
1.269 - AssemblerX86 masm;
1.270 -
1.271 - __ push(ebx);
1.272 - __ push(eax);
1.273 - __ push(edx);
1.274 - __ push(ecx); // We reference this in the body of this routine.
1.275 -
1.276 - Label toobig;
1.277 -
1.278 - // Calculate how many cells will be needed.
1.279 - __ movl(edx, Operand(stk, 0)); // Dimension's last count.
1.280 - __ cmpl(edx, 0);
1.281 - __ j(less, &toobig);
1.282 - __ movl(eax, 1); // Position at second to last dimension.
1.283 -
1.284 - Label loop, done;
1.285 - __ bind(&loop);
1.286 - __ cmpl(eax, Operand(esp, 0)); // Compare to # of dimensions.
1.287 - __ j(not_below, &done); // End loop if done.
1.288 - __ movl(ecx, Operand(stk, eax, ScaleFour)); // Get dimension size.
1.289 - __ cmpl(ecx, 0);
1.290 - __ j(less, &toobig);
1.291 - __ imull(edx, ecx); // Multiply by size.
1.292 - __ j(overflow, &toobig);
1.293 - __ addl(eax, 1); // Increment dimension cursor.
1.294 - __ addl(edx, ecx); // Add indirection vector size.
1.295 - __ j(overflow, &toobig);
1.296 - __ jmp(&loop);
1.297 - __ bind(&done);
1.298 -
1.299 - // Test if we have space for the size in edx.
1.300 - __ movl(eax, Operand(info, AMX_INFO_HEAP));
1.301 - __ imull(edx, edx, 4);
1.302 - __ j(overflow, &toobig);
1.303 - __ addl(eax, edx);
1.304 - __ j(overflow, &toobig);
1.305 - __ cmpl(eax, Operand(info, AMX_INFO_DATASIZE));
1.306 - __ j(not_above, &toobig);
1.307 - __ addl(eax, dat);
1.308 - __ cmpl(eax, stk);
1.309 - __ j(not_below, &toobig);
1.310 - __ shrl(edx, 2); // Undo the imull.
1.311 -
1.312 - // Allocate on the plugin heap.
1.313 - __ movl(eax, Operand(info, AMX_INFO_HEAP)); // Get heap pointer.
1.314 - __ lea(ebx, Operand(eax, edx, ScaleFour)); // New heap pointer.
1.315 - __ movl(Operand(info, AMX_INFO_HEAP), ebx); // Store back.
1.316 - __ push(eax); // Save on the stack.
1.317 -
1.318 - Label extern_error;
1.319 - EmitTrackerPushReg(masm, edx, &extern_error);
1.320 -
1.321 - // Call to C++ to generate indirection vectors.
1.322 - __ lea(ebx, Operand(dat, eax, NoScale)); // relocate eax
1.323 - __ push(Operand(esp, 8)); // push autozero
1.324 - __ push(Operand(esp, 8)); // push dimension count
1.325 - __ push(stk); // push dimension array
1.326 - __ push(ebx);
1.327 - __ call(ExternalAddress((void *)GenerateArrayIndirectionVectors));
1.328 - __ addl(esp, 4 * sizeof(void *));
1.329 -
1.330 - __ pop(eax); // Restore array pointer.
1.331 - __ pop(ecx); // Restore dimension count.
1.332 - __ lea(stk, Operand(stk, ecx, ScaleFour, -4)); // Pop params off the stack.
1.333 - __ movl(Operand(stk, 0), eax); // Store back array pointer.
1.334 - __ pop(edx);
1.335 - __ pop(eax);
1.336 - __ pop(ebx);
1.337 - __ ret();
1.338 -
1.339 - __ bind(&toobig);
1.340 - __ movl(eax, SP_ERROR_ARRAY_TOO_BIG);
1.341 - __ jmp(ExternalAddress(ret));
1.342 -
1.343 - __ bind(&extern_error);
1.344 - __ jmp(ExternalAddress(ret));
1.345 -
1.346 - return LinkCode(masm);
1.347 -}
1.348 -
1.349 static void *
1.350 GenerateEntry(void **retp)
1.351 {
1.352 @@ -1886,7 +1833,6 @@
1.353 JITX86::JITX86()
1.354 {
1.355 m_pJitEntry = NULL;
1.356 - m_pJitGenArray = NULL;
1.357 }
1.358
1.359 bool JITX86::InitializeJIT()
1.360 @@ -1897,10 +1843,6 @@
1.361 if (!m_pJitEntry)
1.362 return false;
1.363
1.364 - m_pJitGenArray = GenerateGenArrayIntrinsic(m_pJitReturn);
1.365 - if (!m_pJitGenArray)
1.366 - return false;
1.367 -
1.368 MacroAssemblerX86 masm;
1.369 MacroAssemblerX86::GenerateFeatureDetection(masm);
1.370 void *code = LinkCode(masm);
2.1 --- a/sourcepawn/jit/x86/jit_x86.h Sat Aug 10 12:22:09 2013 -0400
2.2 +++ b/sourcepawn/jit/x86/jit_x86.h Sat Aug 10 21:23:02 2013 -0700
2.3 @@ -175,9 +175,6 @@
2.4 int InvokeFunction(BaseRuntime *runtime, JitFunction *fn, cell_t *result);
2.5
2.6 public:
2.7 - ExternalAddress GetGenArrayIntrinsic() {
2.8 - return ExternalAddress(m_pJitGenArray);
2.9 - }
2.10 ExternalAddress GetUniversalReturn() {
2.11 return ExternalAddress(m_pJitReturn);
2.12 }
2.13 @@ -187,7 +184,6 @@
2.14 private:
2.15 void *m_pJitEntry; /* Entry function */
2.16 void *m_pJitReturn; /* Universal return address */
2.17 - void *m_pJitGenArray; /* Generates an array */
2.18 };
2.19
2.20 const Register pri = eax;
2.21 @@ -198,6 +194,18 @@
2.22 const Register info = esi;
2.23 const Register frm = ebx;
2.24
2.25 +struct InfoVars {
2.26 + ucell_t frm;
2.27 + ucell_t hp;
2.28 + cell_t *rval;
2.29 + sp_context_t *ctx;
2.30 + uint8_t *stp;
2.31 + ucell_t cip;
2.32 + size_t data_size;
2.33 + uint8_t *memory;
2.34 + void *esp;
2.35 +};
2.36 +
2.37 #define AMX_NUM_INFO_VARS 9
2.38
2.39 #define AMX_INFO_FRAME 0 //(same thing as above)
