Added a mark-sweep garbage collector.
Added a mark-sweep garbage collector.
1.1 --- a/jitcraft/Array.cpp Sat Jul 07 01:45:33 2012 -0400
1.2 +++ b/jitcraft/Array.cpp Sat Jul 07 17:11:31 2012 -0700
1.3 @@ -45,19 +45,21 @@
1.4 if (type->contained()->primitive() == PrimitiveType_Char)
1.5 return sizeof(Array) + elements * sizeof(char);
1.6 assert(type->contained()->isFloat());
1.7 - return sizeof(Array) + elements * sizeof(float);
1.8 + return elements * sizeof(float);
1.9 }
1.10
1.11 Array *
1.12 Array::New(Zone *zone, Handle<ArrayMap> map, uint32 elements, Heap::Tenure tenure)
1.13 {
1.14 - size_t bytes = SizeFor(map->type(), elements);
1.15 + size_t elementBytes = SizeFor(map->type(), elements);
1.16 + size_t bytes = sizeof(Array) + elementBytes;
1.17 Local<Array> array(zone, Array::cast(zone->allocate(map, bytes, tenure)));
1.18 if (!array)
1.19 return NULL;
1.20
1.21 array->nelements_ = elements;
1.22 array->elements_ = VoidPtr((Array *)array + 1);
1.23 + memset(array->elements_, 0, elementBytes);
1.24 return array;
1.25 }
1.26
2.1 --- a/jitcraft/Code.h Sat Jul 07 01:45:33 2012 -0400
2.2 +++ b/jitcraft/Code.h Sat Jul 07 17:11:31 2012 -0700
2.3 @@ -99,6 +99,9 @@
2.4 static inline size_t offsetOfObjects() {
2.5 return offsetof(Code, objects_);
2.6 }
2.7 + static inline size_t offsetOfSafepoints() {
2.8 + return offsetof(Code, safepoints_);
2.9 + }
2.10 // End GC Descriptor.
2.11
2.12 static Code *cast(Object *obj) {
3.1 --- a/jitcraft/FixedArray.cpp Sat Jul 07 01:45:33 2012 -0400
3.2 +++ b/jitcraft/FixedArray.cpp Sat Jul 07 17:11:31 2012 -0700
3.3 @@ -38,6 +38,7 @@
3.4 return NULL;
3.5
3.6 array->length_ = length;
3.7 + memset(array->vec(), 0, length * sizeof(Object *));
3.8 return array;
3.9 }
3.10
3.11 @@ -55,6 +56,7 @@
3.12 return NULL;
3.13
3.14 array->length_ = length;
3.15 + memset(array->raw(), 0, length);
3.16 return array;
3.17 }
3.18
4.1 --- a/jitcraft/Handles-inl.h Sat Jul 07 01:45:33 2012 -0400
4.2 +++ b/jitcraft/Handles-inl.h Sat Jul 07 17:11:31 2012 -0700
4.3 @@ -1,27 +1,27 @@
4.4 -/*
4.5 - * Copyright (C) 2012 David Anderson
4.6 - *
4.7 - * This file is part of JITCraft.
4.8 - *
4.9 - * JITCraft is free software: you can redistribute it and/or modify it under
4.10 - * the terms of the GNU General Public License as published by the Free
4.11 - * Software Foundation, either version 3 of the License, or (at your option)
4.12 - * any later version.
4.13 - *
4.14 - * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
4.15 - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
4.16 - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
4.17 - *
4.18 - * You should have received a copy of the GNU General Public License along with
4.19 - * JITCraft. If not, see http://www.gnu.org/licenses/.
4.20 - */
4.21 -#ifndef _include_jitcraft_rooting_inl_h_
4.22 -#define _include_jitcraft_rooting_inl_h_
4.23 -
4.24 -#include "Heap.h"
4.25 -
4.26 -namespace jc {
4.27 -
4.28 +/*
4.29 + * Copyright (C) 2012 David Anderson
4.30 + *
4.31 + * This file is part of JITCraft.
4.32 + *
4.33 + * JITCraft is free software: you can redistribute it and/or modify it under
4.34 + * the terms of the GNU General Public License as published by the Free
4.35 + * Software Foundation, either version 3 of the License, or (at your option)
4.36 + * any later version.
4.37 + *
4.38 + * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
4.39 + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
4.40 + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
4.41 + *
4.42 + * You should have received a copy of the GNU General Public License along with
4.43 + * JITCraft. If not, see http://www.gnu.org/licenses/.
4.44 + */
4.45 +#ifndef _include_jitcraft_rooting_inl_h_
4.46 +#define _include_jitcraft_rooting_inl_h_
4.47 +
4.48 +#include "Heap.h"
4.49 +
4.50 +namespace jc {
4.51 +
4.52 template <typename T>
4.53 Local<T>::Local(Environment *env)
4.54 : zone_(env->zone()), thing_(NULL)
4.55 @@ -55,8 +55,8 @@
4.56 Local<T>::~Local()
4.57 {
4.58 zone_->deregisterThingLocal(toRaw(), prev_);
4.59 -}
4.60 -
4.61 -}
4.62 -
4.63 -#endif // _include_jitcraft_rooting_inl_h_
4.64 +}
4.65 +
4.66 +}
4.67 +
4.68 +#endif // _include_jitcraft_rooting_inl_h_
5.1 --- a/jitcraft/Handles.h Sat Jul 07 01:45:33 2012 -0400
5.2 +++ b/jitcraft/Handles.h Sat Jul 07 17:11:31 2012 -0700
5.3 @@ -142,8 +142,7 @@
5.4 }
5.5
5.6 public:
5.7 - ScopedRoot()
5.8 - { }
5.9 + ScopedRoot();
5.10
5.11 // This is implemented in Zone.h.
5.12 inline ScopedRoot(T *thing);
5.13 @@ -166,7 +165,7 @@
5.14 return *thingp_;
5.15 }
5.16 operator bool () const {
5.17 - return *thingp_;
5.18 + return !!*thingp_;
5.19 }
5.20 T **address() const {
5.21 return thingp_;
6.1 --- a/jitcraft/Heap.cpp Sat Jul 07 01:45:33 2012 -0400
6.2 +++ b/jitcraft/Heap.cpp Sat Jul 07 17:11:31 2012 -0700
6.3 @@ -29,7 +29,9 @@
6.4
6.5 Heap::Heap(Zone *zone)
6.6 : zone_(zone),
6.7 - markStack_(NULL)
6.8 + markStack_(NULL),
6.9 + oldSpace_(zone),
6.10 + mapSpace_(zone)
6.11 {
6.12 }
6.13
6.14 @@ -122,6 +124,7 @@
6.15 }
6.16
6.17 void pushBlack(Object *ptr) {
6.18 + assert(HeapChunk::IsBlack(HeapChunk::GetMarkBit(ptr)));
6.19 if (top_ >= end_) {
6.20 MarkBit bit = HeapChunk::GetMarkBit(ptr);
6.21 HeapChunk::BlackToGrey(bit);
6.22 @@ -135,14 +138,16 @@
6.23 assert(!empty());
6.24 return *--top_;
6.25 }
6.26 -
6.27 bool overflowed() const {
6.28 return overflowed_;
6.29 }
6.30 -
6.31 bool empty() const {
6.32 return base_ == top_;
6.33 }
6.34 + void reset() {
6.35 + assert(empty());
6.36 + overflowed_ = true;
6.37 + }
6.38 };
6.39 } // namespace jc
6.40
6.41 @@ -211,14 +216,58 @@
6.42 }
6.43 };
6.44
6.45 +typedef ChunkObjectIteratorBase<ObjectSizeEvaluator> ChunkObjectIterator;
6.46 +
6.47 +static inline bool
6.48 +EnqueueGreyObject(MarkStack &stack, Object *obj)
6.49 +{
6.50 + assert(!stack.overflowed());
6.51 + MarkBit bit = HeapChunk::GetMarkBit(obj);
6.52 + if (HeapChunk::IsGrey(bit)) {
6.53 + HeapChunk::GreyToBlack(bit);
6.54 + stack.pushBlack(obj);
6.55 + return !stack.overflowed();
6.56 + }
6.57 + return true;
6.58 +}
6.59 +
6.60 static void
6.61 -EnqueueGreyObjectsInChunks(MarkStack &stack, HeapChunk *chunk)
6.62 +EnqueueGreyObjectsInChunks(MarkStack &stack, ChunkObjectIterator iter)
6.63 {
6.64 + for (Object *obj = iter.next(); obj; obj = iter.next()) {
6.65 + if (!EnqueueGreyObject(stack, obj))
6.66 + return;
6.67 + }
6.68 +}
6.69 +
6.70 +static void
6.71 +EnqueueGreyObjectsInLargeChunks(MarkStack &stack, HeapChunk *first)
6.72 +{
6.73 + for (HeapChunk *chunk = first; chunk; chunk = chunk->next()) {
6.74 + Object *obj = reinterpret_cast<Object *>(chunk->first());
6.75 + if (!EnqueueGreyObject(stack, obj))
6.76 + return;
6.77 + }
6.78 }
6.79
6.80 void
6.81 Heap::findGreyObjects(MarkStack &stack)
6.82 {
6.83 + EnqueueGreyObjectsInChunks(stack, lifoSpace_.begin(ObjectSizeEvaluator()));
6.84 + if (stack.overflowed())
6.85 + return;
6.86 +
6.87 + EnqueueGreyObjectsInChunks(stack, mapSpace_.begin(ObjectSizeEvaluator()));
6.88 + if (stack.overflowed())
6.89 + return;
6.90 +
6.91 + EnqueueGreyObjectsInChunks(stack, oldSpace_.begin(ObjectSizeEvaluator()));
6.92 + if (stack.overflowed())
6.93 + return;
6.94 +
6.95 + EnqueueGreyObjectsInLargeChunks(stack, largeSpace_.first());
6.96 + if (stack.overflowed())
6.97 + return;
6.98 }
6.99
6.100 void
6.101 @@ -234,13 +283,83 @@
6.102 assert(stack.empty());
6.103
6.104 // If the stack overflowed, we need to look through the entire heap for
6.105 - // grey objects to rescan.
6.106 - //
6.107 + // grey objects to rescan. We keep doing this until there are no grey
6.108 + // objects left.
6.109 + while (stack.overflowed()) {
6.110 + stack.reset();
6.111 + findGreyObjects(stack);
6.112 + ProcessMarkStack(stack);
6.113 + }
6.114 +}
6.115 +
6.116 +// Use the mark bits to scan for live objects, and mark the regions in
6.117 +// between them as free.
6.118 +static void
6.119 +SweepOldSpaceChunk(OldSpace *space, HeapChunk *chunk)
6.120 +{
6.121 + uint32 *markbits = chunk->markBits();
6.122 + uint32 *markbitsEnd = markbits + HeapChunk::kMarkWords;
6.123 + Address freeStart = chunk->first();
6.124 +
6.125 + // Scoot up to the first usable markword. Since memory is reserved for
6.126 + // mark bits, not all addresses are allocatable, so quite a few of the
6.127 + // initial mark bits are wasted. (1KB of mark bytes on x86.)
6.128 + markbits = HeapChunk::GetMarkBit(reinterpret_cast<Object *>(chunk->first())).word();
6.129 +
6.130 + uint32 bits = *markbits;
6.131 + for (;;) {
6.132 + while (bits) {
6.133 + uint32 bit = FindRightmostBit(bits);
6.134 + Address object = HeapChunk::FromMarkBit(chunk, markbits, bit);
6.135 + if (freeStart != object)
6.136 + space->markRegionFree(freeStart, object - freeStart);
6.137 + Object *obj = reinterpret_cast<Object *>(object);
6.138 + freeStart = object + SizeOfObject(obj, obj->map());
6.139 +
6.140 + bits &= ~(1 << bit);
6.141 + if (bits == HeapChunk::kMarkBitsPerWord - 1) {
6.142 + // This bit is the last bit on this word, so bump up to the
6.143 + // next word and exclude the first bit.
6.144 + markbits++;
6.145 + assert(markbits < markbitsEnd);
6.146 + bits = *markbits & ~(1 << 0);
6.147 + }
6.148 + }
6.149 + markbits++;
6.150 + bits = *markbits;
6.151 + if (markbits >= markbitsEnd)
6.152 + break;
6.153 + }
6.154 +
6.155 + if (freeStart == chunk->base())
6.156 + chunk->setEmpty();
6.157 + else if (freeStart != chunk->end())
6.158 + space->markRegionFree(freeStart, chunk->end() - freeStart);
6.159 +}
6.160 +
6.161 +static void
6.162 +SweepOldSpace(OldSpace *space)
6.163 +{
6.164 + space->resetFreeList();
6.165 + for (HeapChunk *chunk = space->first(); chunk; chunk = chunk->next()) {
6.166 + SweepOldSpaceChunk(space, chunk);
6.167 + chunk->resetMarkBits();
6.168 + }
6.169 + space->freeUnusedPages();
6.170 +}
6.171 +
6.172 +void
6.173 +Heap::sweep()
6.174 +{
6.175 + // Note: we don't bother sweeping the LIFO space.
6.176 + SweepOldSpace(&mapSpace_);
6.177 + SweepOldSpace(&oldSpace_);
6.178 + largeSpace_.sweep();
6.179 }
6.180
6.181 void
6.182 Heap::reclaim()
6.183 {
6.184 - // Full memory reclamation.
6.185 + mark();
6.186 + sweep();
6.187 }
6.188 -
7.1 --- a/jitcraft/Heap.h Sat Jul 07 01:45:33 2012 -0400
7.2 +++ b/jitcraft/Heap.h Sat Jul 07 17:11:31 2012 -0700
7.3 @@ -34,57 +34,34 @@
7.4 // four spaces:
7.5 //
7.6 // LIFO Space: Used for objects that may be stack-allocated.
7.7 -// New Space: Used for objects that are expected to be short-lived.
7.8 // Old Space: Used for objects that are expected to be long-lived, or that
7.9 // survive one collection.
7.10 // Large Space: Used for objects that are so large they should never be
7.11 // moved between spaces.
7.12 //
7.13 +// ALLOCATING AN OBJECT DOES NOT ZERO ITS MEMORY.
7.14 +// ALLOCATING AN OBJECT DOES NOT ZERO ITS MEMORY!
7.15 +//
7.16 +// ---------------------------------------------------------------------------
7.17 +//
7.18 // The following is a high-level overview of how garbage collection works in
7.19 // Keima. Detailed notes are present in Heap.cpp.
7.20 //
7.21 -// THE TRI-COLOR INVARIANT: Via an object's mark bits, we maintain the property
7.22 -// that an object that has been fully marked may never reference a dead object.
7.23 -// Objects exist in three states during collection: unmarked (white), scheduled
7.24 -// for marking (grey), and fully marked (black).
7.25 +// THE TRI-COLOR INVARIANT: Via an object's mark bits, we maintain the
7.26 +// property that an object that has been fully marked may never reference a
7.27 +// dead object. Objects exist in three states during collection: unmarked
7.28 +// (white), scheduled for marking (grey), and fully marked (black).
7.29 //
7.30 -// When initiating collection, all objects are white. Objects in the root set
7.31 -// are made grey. The marking process takes grey objects and marks that each
7.32 -// child object is live: if the child is white, it becomes grey. Once all
7.33 -// children are grey or black, the parent grey object can become black, as it
7.34 -// has been fully traversed.
7.35 +// When initiating collection, all objects are white. Objects in the root
7.36 +// set are made grey. The marking process takes grey objects and marks that
7.37 +// each child object is live: if the child is white, it becomes grey. Once
7.38 +// all children are grey or black, the parent grey object can become black,
7.39 +// as it has been fully traversed.
7.40 //
7.41 // The result of this process is that when there are no more grey objects,
7.42 // then we are guaranteed that all black objects are live, and all white
7.43 // objects are dead.
7.44 //
7.45 -// MINOR COLLECTIONS (also called "scavenges") are usually performed when the
7.46 -// new space reaches maximum capacity. During a minor collection, all live
7.47 -// objects in the new space are moved into the old space. Then, the new space
7.48 -// is reset such that it is empty.
7.49 -//
7.50 -// The new is split into contiguous halves. When both halves reach maximum
7.51 -// capacity, a scavenge is performed on the older half. Thus, allocation is
7.52 -// only ever performed in one half at a time, to give newer objects a chance
7.53 -// to die.
7.54 -//
7.55 -// Scavenges are beneficial for two major reasons:
7.56 -// (1) Allocation from the new space is just a pointer bump.
7.57 -// (2) Only the root set and remembered sets need to visited.
7.58 -//
7.59 -// The "remembered set" is a list of all pointers that reach into the new
7.60 -// space, but come from the Old or Large spaces. The remembered set is
7.61 -// maintained via write barriers, and if it becomes full, entire chunks
7.62 -// are marked as requiring traversal.
7.63 -//
7.64 -// The root and remembered sets must be visited twice: once to determine
7.65 -// which objects in the new space are live, and again to update any
7.66 -// references after the new space has been reclaimed.
7.67 -//
7.68 -// Live objects in the LIFO space are always deeply traversed for pointers
7.69 -// into the new space, since it would be too difficult to maintain a working
7.70 -// remembered set.
7.71 -//
7.72 // MAJOR COLLECTIONS are performed periodically, when either a given memory
7.73 // threshold has been reached, or when requested by the host application.
7.74 // They are also performed if no memory can be allocated.
7.75 @@ -132,8 +109,10 @@
7.76 inline Address newStackObject(size_t bytes);
7.77 inline Address newObject(size_t bytes, Tenure pretenure);
7.78
7.79 + private:
7.80 + void findGreyObjects(MarkStack &stack);
7.81 void mark();
7.82 - void findGreyObjects(MarkStack &stack);
7.83 + void sweep();
7.84
7.85 public:
7.86 Heap(Zone *zone);
8.1 --- a/jitcraft/Interpreter.cpp Sat Jul 07 01:45:33 2012 -0400
8.2 +++ b/jitcraft/Interpreter.cpp Sat Jul 07 17:11:31 2012 -0700
8.3 @@ -365,14 +365,7 @@
8.4
8.5 case OP_CALLGLOBAL:
8.6 {
8.7 - class BlahVisitor : public VirtualObjectVisitor
8.8 - {
8.9 - public:
8.10 - void visit(Object **ptr) {
8.11 - printf("Visiting %p %p\n", ptr, *ptr);
8.12 - }
8.13 - } blah;
8.14 - zone->stack().accept(&blah);
8.15 + zone->heap().reclaim();
8.16 unsigned index = READ_UINT32(top.pc);
8.17 Local<Function> newFun(zone, top.fp->code()->module()->getFunction(index));
8.18 if (!zone->stack().pushFrame(newFun)) {
9.1 --- a/jitcraft/ObjectVisitor.h Sat Jul 07 01:45:33 2012 -0400
9.2 +++ b/jitcraft/ObjectVisitor.h Sat Jul 07 17:11:31 2012 -0700
9.3 @@ -65,6 +65,7 @@
9.4 visitField(obj, Code::offsetOfBytecode());
9.5 visitField(obj, Code::offsetOfLocals());
9.6 visitField(obj, Code::offsetOfObjects());
9.7 + visitField(obj, Code::offsetOfSafepoints());
9.8 return sizeof(Code);
9.9 }
9.10
9.11 @@ -116,7 +117,7 @@
9.12 return iterateFixedArray(FixedArray::cast(obj));
9.13
9.14 case MapKind_ByteArray:
9.15 - return iterateByteArray(ByteArray::cast(obj));
9.16 + return Align(iterateByteArray(ByteArray::cast(obj)), kObjectAlignment);
9.17
9.18 case MapKind_Code:
9.19 return iterateCode(obj);
9.20 @@ -131,7 +132,7 @@
9.21 return iterateFunction(obj);
9.22
9.23 case MapKind_Array:
9.24 - return iterateArray(Array::cast(obj), ArrayMap::cast(map));
9.25 + return Align(iterateArray(Array::cast(obj), ArrayMap::cast(map)), kObjectAlignment);
9.26
9.27 case MapKind_MetaArray:
9.28 return iterateArrayMap(ArrayMap::cast(obj));
9.29 @@ -143,6 +144,54 @@
9.30 }
9.31 };
9.32
9.33 +static inline size_t
9.34 +SizeOfObject(Object *obj, Map *map)
9.35 +{
9.36 + switch (map->kind()) {
9.37 + case MapKind_Type:
9.38 + return sizeof(Type);
9.39 +
9.40 + case MapKind_FixedArray:
9.41 + return FixedArray::SizeFor(FixedArray::cast(obj)->length());
9.42 +
9.43 + case MapKind_ByteArray:
9.44 + return Align(ByteArray::SizeFor(ByteArray::cast(obj)->length()), kObjectAlignment);
9.45 +
9.46 + case MapKind_Code:
9.47 + return sizeof(Code);
9.48 +
9.49 + case MapKind_Reference:
9.50 + return sizeof(Reference);
9.51 +
9.52 + case MapKind_Module:
9.53 + return sizeof(Module);
9.54 +
9.55 + case MapKind_Function:
9.56 + return sizeof(Function);
9.57 +
9.58 + case MapKind_Array:
9.59 + return Align(Array::SizeOf(Array::cast(obj)), kObjectAlignment);
9.60 +
9.61 + case MapKind_MetaArray:
9.62 + return sizeof(ArrayMap);
9.63 +
9.64 + case MapKind_FreeRegion:
9.65 + return *reinterpret_cast<size_t *>(reinterpret_cast<Address>(obj + 1));
9.66 +
9.67 + default:
9.68 + assert(map->kind() == MapKind_Meta);
9.69 + return sizeof(Map);
9.70 + }
9.71 +}
9.72 +
9.73 +class ObjectSizeEvaluator
9.74 +{
9.75 + public:
9.76 + size_t operator()(Object *obj, Map *map) {
9.77 + return SizeOfObject(obj, map);
9.78 + }
9.79 +};
9.80 +
9.81 }
9.82
9.83 #endif // _include_jc_object_visitor_h_
10.1 --- a/jitcraft/Objects.h Sat Jul 07 01:45:33 2012 -0400
10.2 +++ b/jitcraft/Objects.h Sat Jul 07 17:11:31 2012 -0700
10.3 @@ -45,6 +45,7 @@
10.4 MapKind_Reference,
10.5 MapKind_Module,
10.6 MapKind_Function,
10.7 + MapKind_FreeRegion, // Special map denoting freed space.
10.8
10.9 // Used to represent Arrays. This does not have a canonical map.
10.10 MapKind_Array,
10.11 @@ -79,8 +80,7 @@
10.12 inline void write(Zone *zone, Object **ptr, Object *value);
10.13
10.14 static inline size_t offsetOfMap() {
10.15 - // Wtf? Clang won't accept offsetof(Object, map_) here...
10.16 - return 0;
10.17 + return offsetof(Object, map_);
10.18 }
10.19 };
10.20
11.1 --- a/jitcraft/Platform-Windows.cpp Sat Jul 07 01:45:33 2012 -0400
11.2 +++ b/jitcraft/Platform-Windows.cpp Sat Jul 07 17:11:31 2012 -0700
11.3 @@ -63,11 +63,11 @@
11.4 }
11.5
11.6 static Address
11.7 -MapMemoryAt(Address address, size_t amount, MapMode mode, Protection prot_)
11.8 +MapMemoryAt(Address address, size_t amount, OS::MapMode mode, OS::Protection prot_)
11.9 {
11.10 DWORD bits = ToProtBits(prot_);
11.11 DWORD flags = MEM_RESERVE;
11.12 - if (mode == Map_Commit)
11.13 + if (mode == OS::Map_Commit)
11.14 flags |= MEM_COMMIT;
11.15
11.16 Address x = (Address)VirtualAlloc((LPVOID)address, amount, flags, bits);
12.1 --- a/jitcraft/Spaces-inl.h Sat Jul 07 01:45:33 2012 -0400
12.2 +++ b/jitcraft/Spaces-inl.h Sat Jul 07 17:11:31 2012 -0700
12.3 @@ -49,7 +49,7 @@
12.4 if (cursor_ + nbytes > last_->end())
12.5 return allocateSlow(nbytes);
12.6
12.7 - Address addr = cursor_ + nbytes;
12.8 + Address addr = cursor_;
12.9 cursor_ += nbytes;
12.10 return addr;
12.11 }
13.1 --- a/jitcraft/Spaces.cpp Sat Jul 07 01:45:33 2012 -0400
13.2 +++ b/jitcraft/Spaces.cpp Sat Jul 07 17:11:31 2012 -0700
13.3 @@ -16,6 +16,7 @@
13.4 * You should have received a copy of the GNU General Public License along with
13.5 * JITCraft. If not, see http://www.gnu.org/licenses/.
13.6 */
13.7 +#include "Zone.h"
13.8 #include "Spaces.h"
13.9 #include "Platform.h"
13.10 #include "Spaces-inl.h"
13.11 @@ -66,8 +67,79 @@
13.12 OS::UnmapAlignedMemory(chunk->base(), chunk->size());
13.13 }
13.14
13.15 -OldSpace::OldSpace()
13.16 - : first_(NULL),
13.17 +void
13.18 +HeapChunk::resetMarkBits()
13.19 +{
13.20 + memset(markBits(), 0, kMarkBytes);
13.21 +}
13.22 +
13.23 +FreeList::FreeList()
13.24 + : small_objects_(NULL),
13.25 + medium_objects_(NULL),
13.26 + large_objects_(NULL),
13.27 + huge_objects_(NULL)
13.28 +{
13.29 +}
13.30 +
13.31 +void
13.32 +FreeList::add(FreedObject *obj)
13.33 +{
13.34 + if (obj->size() < kMinObjectSize)
13.35 + return;
13.36 +
13.37 + FreedObject **list;
13.38 + if (obj->size() < kMaxSmallObject)
13.39 + list = &small_objects_;
13.40 + else if (obj->size() < kMaxMediumObject)
13.41 + list = &medium_objects_;
13.42 + else if (obj->size() < kMaxLargeObject)
13.43 + list = &large_objects_;
13.44 + else
13.45 + list = &huge_objects_;
13.46 +
13.47 + if (*list)
13.48 + (*list)->setNext(obj);
13.49 + *list = obj;
13.50 +}
13.51 +
13.52 +FreedObject *
13.53 +FreeList::pick(size_t size)
13.54 +{
13.55 + if (size < kMinObjectSize && small_objects_) {
13.56 + FreedObject *obj = small_objects_;
13.57 + small_objects_ = small_objects_->next();
13.58 + return obj;
13.59 + }
13.60 + if (size < kMaxSmallObject && medium_objects_) {
13.61 + FreedObject *obj = medium_objects_;
13.62 + medium_objects_ = medium_objects_->next();
13.63 + return obj;
13.64 + }
13.65 + if (size < kMaxMediumObject && large_objects_) {
13.66 + FreedObject *obj = large_objects_;
13.67 + large_objects_ = large_objects_->next();
13.68 + return obj;
13.69 + }
13.70 + if (huge_objects_) {
13.71 + FreedObject *obj = huge_objects_;
13.72 + huge_objects_ = huge_objects_->next();
13.73 + return obj;
13.74 + }
13.75 + return NULL;
13.76 +}
13.77 +
13.78 +void
13.79 +FreeList::reset()
13.80 +{
13.81 + small_objects_ = NULL;
13.82 + medium_objects_ = NULL;
13.83 + large_objects_ = NULL;
13.84 + huge_objects_ = NULL;
13.85 +}
13.86 +
13.87 +OldSpace::OldSpace(Zone *zone)
13.88 + : zone_(zone),
13.89 + first_(NULL),
13.90 last_(NULL)
13.91 {
13.92 }
13.93 @@ -82,12 +154,41 @@
13.94 return !!first_;
13.95 }
13.96
13.97 +OldSpace::~OldSpace()
13.98 +{
13.99 + while (first_) {
13.100 + HeapChunk *temp = first_->next();
13.101 + HeapChunk::Destroy(first_);
13.102 + first_ = temp;
13.103 + }
13.104 +}
13.105 +
13.106 Address
13.107 OldSpace::allocateSlow(size_t nbytes)
13.108 {
13.109 - HeapChunk *chunk = HeapChunk::New(last_, HeapChunk::kDefaultSize);
13.110 - if (!chunk)
13.111 - return NULL;
13.112 + if (FreedObject *obj = freelist_.pick(nbytes)) {
13.113 + Address address = reinterpret_cast<Address>(obj);
13.114 + size_t actual = obj->size();
13.115 + assert(actual >= nbytes);
13.116 + if (actual != nbytes)
13.117 + markRegionFree(address + nbytes, actual - nbytes);
13.118 +
13.119 + return address;
13.120 + }
13.121 +
13.122 + // Free unused stuff at the end of the current chunk.
13.123 + if (cursor_ != last_->end())
13.124 + markRegionFree(cursor_, last_->end() - cursor_);
13.125 +
13.126 + HeapChunk *chunk;
13.127 + if (last_->next()) {
13.128 + // The next chunk is guaranteed to be empty.
13.129 + chunk = last_->next();
13.130 + } else {
13.131 + chunk = HeapChunk::New(last_, HeapChunk::kDefaultSize);
13.132 + if (!chunk)
13.133 + return NULL;
13.134 + }
13.135
13.136 last_ = chunk;
13.137 cursor_ = chunk->first();
13.138 @@ -99,13 +200,49 @@
13.139 return address;
13.140 }
13.141
13.142 -OldSpace::~OldSpace()
13.143 +void
13.144 +OldSpace::markRegionFree(Address start, size_t length)
13.145 +{
13.146 + FreedObject *obj = reinterpret_cast<FreedObject *>(start);
13.147 + new (obj) FreedObject(zone_->getMap(MapKind_FreeRegion), length);
13.148 +
13.149 + if (length < sizeof(FreedObject)) {
13.150 + // Can't encode a list in this object, but it's marked free, so we're done.
13.151 + return;
13.152 + }
13.153 +
13.154 + freelist_.add(obj);
13.155 +}
13.156 +
13.157 +void
13.158 +OldSpace::freeUnusedPages()
13.159 {
13.160 - while (first_) {
13.161 - HeapChunk *temp = first_->next();
13.162 - HeapChunk::Destroy(first_);
13.163 - first_ = temp;
13.164 + HeapChunk **prev = &first_;
13.165 + HeapChunk *empty = NULL;
13.166 + while (*prev) {
13.167 + HeapChunk *chunk = *prev;
13.168 + if (chunk->isEmpty()) {
13.169 + *prev = chunk->next();
13.170 + if (empty) {
13.171 + // Remove this chunk.
13.172 + HeapChunk::Destroy(chunk);
13.173 + continue;
13.174 + }
13.175 + empty = chunk;
13.176 + }
13.177 + prev = chunk->nextp();
13.178 }
13.179 +
13.180 + if (empty) {
13.181 + *empty->nextp() = NULL;
13.182 + *last_->nextp() = empty;
13.183 + }
13.184 +}
13.185 +
13.186 +void
13.187 +OldSpace::resetFreeList()
13.188 +{
13.189 + freelist_.reset();
13.190 }
13.191
13.192 LargeObjectSpace::LargeObjectSpace()
13.193 @@ -146,3 +283,25 @@
13.194 return chunk->first();
13.195 }
13.196
13.197 +void
13.198 +LargeObjectSpace::sweep()
13.199 +{
13.200 + if (!first_)
13.201 + return;
13.202 +
13.203 + HeapChunk **pprev = &first_;
13.204 + do {
13.205 + HeapChunk *chunk = *pprev;
13.206 + Object *obj = reinterpret_cast<Object *>(chunk->first());
13.207 + MarkBit bit = HeapChunk::GetMarkBit(obj);
13.208 + if (HeapChunk::IsWhite(bit)) {
13.209 + *pprev = chunk->next();
13.210 + HeapChunk::Destroy(chunk);
13.211 + } else {
13.212 + assert(HeapChunk::IsBlack(bit));
13.213 + HeapChunk::Clear(bit);
13.214 + pprev = chunk->nextp();
13.215 + last_ = chunk;
13.216 + }
13.217 + } while (*pprev);
13.218 +}
14.1 --- a/jitcraft/Spaces.h Sat Jul 07 01:45:33 2012 -0400
14.2 +++ b/jitcraft/Spaces.h Sat Jul 07 17:11:31 2012 -0700
14.3 @@ -25,7 +25,9 @@
14.4
14.5 namespace jc {
14.6
14.7 +class Zone;
14.8 class Object;
14.9 +class Map;
14.10
14.11 static const size_t kObjectAlignment = sizeof(Object *);
14.12
14.13 @@ -58,6 +60,12 @@
14.14 void set() {
14.15 *word_ |= (1 << bit_);
14.16 }
14.17 + void unset() {
14.18 + *word_ &= ~(1 << bit_);
14.19 + }
14.20 + uint32 *word() {
14.21 + return word_;
14.22 + }
14.23 MarkBit next() const {
14.24 if (bit_ == (sizeof(*word_) * 8) - 1)
14.25 return MarkBit(word_ + 1, 0);
14.26 @@ -77,14 +85,10 @@
14.27
14.28 uintptr_t flags_;
14.29
14.30 - private:
14.31 - uint32 *markBits() {
14.32 - return reinterpret_cast<uint32 *>(reinterpret_cast<Address>(this) + kMarkBitsOffset);
14.33 - }
14.34 -
14.35 public:
14.36 enum Flag {
14.37 - HAS_OVERFLOWED_OBJECT = (1 << 0)
14.38 + HAS_OVERFLOWED_OBJECT = (1 << 0),
14.39 + EMPTY = (1 << 1)
14.40 };
14.41
14.42 public:
14.43 @@ -165,6 +169,9 @@
14.44 Address base() {
14.45 return base_;
14.46 }
14.47 + Address address() {
14.48 + return reinterpret_cast<Address>(this);
14.49 + }
14.50 Address end() {
14.51 return end_;
14.52 }
14.53 @@ -174,6 +181,13 @@
14.54 Address first() {
14.55 return reinterpret_cast<Address>(this) + kFirstObject;
14.56 }
14.57 + HeapChunk **nextp() {
14.58 + return &next_;
14.59 + }
14.60 + uint32 *markBits() {
14.61 + return reinterpret_cast<uint32 *>(reinterpret_cast<Address>(this) + kMarkBitsOffset);
14.62 + }
14.63 + void resetMarkBits();
14.64
14.65 // Marking functions.
14.66 static inline MarkBit GetMarkBit(Object *obj) {
14.67 @@ -189,6 +203,14 @@
14.68 uint32 bit = index & (kMarkBitsPerWord - 1);
14.69 return MarkBit(word, bit);
14.70 }
14.71 + static inline Address FromMarkBit(HeapChunk *chunk, uint32 *word, uint32 bit) {
14.72 + // word = bits + (index / kMarkBitsPerWord)
14.73 + // word - bits = index / kMarkBitsPerWord
14.74 + // (word - bits) * kMarkBitsPerWord = index
14.75 + assert(word >= chunk->markBits() && word < chunk->markBits() + kMarkWords);
14.76 + uint32 index = ((word - chunk->markBits()) * kMarkBitsPerWord) + bit;
14.77 + return chunk->address() + (index * kObjectAlignment);
14.78 + }
14.79 static inline bool IsWhite(const MarkBit &bit) {
14.80 assert(bit.isSet() || !bit.next().isSet());
14.81 return !bit.isSet();
14.82 @@ -207,8 +229,16 @@
14.83 bit.set();
14.84 }
14.85 static inline void BlackToGrey(MarkBit &bit) {
14.86 + assert(IsBlack(bit));
14.87 + bit.next().set();
14.88 + }
14.89 + static inline void GreyToBlack(MarkBit &bit) {
14.90 assert(IsGrey(bit));
14.91 - bit.next().set();
14.92 + bit.next().unset();
14.93 + }
14.94 + static inline void Clear(MarkBit &bit) {
14.95 + bit.unset();
14.96 + bit.next().unset();
14.97 }
14.98
14.99 bool hasOverflowedObject() {
14.100 @@ -217,12 +247,100 @@
14.101 void setHasOverflowedObject() {
14.102 flags_ |= HAS_OVERFLOWED_OBJECT;
14.103 }
14.104 + bool isEmpty() {
14.105 + return !!(flags_ & EMPTY);
14.106 + }
14.107 + void setEmpty() {
14.108 + flags_ |= EMPTY;
14.109 + }
14.110 +};
14.111 +
14.112 +template <typename SizeEvaluator>
14.113 +class ChunkObjectIteratorBase
14.114 +{
14.115 + HeapChunk *chunk_;
14.116 + Address top_;
14.117 + SizeEvaluator evaluator_;
14.118 + Address current_;
14.119 +
14.120 + public:
14.121 + ChunkObjectIteratorBase(HeapChunk *start, Address top, const SizeEvaluator &evaluator)
14.122 + : chunk_(start),
14.123 + top_(top),
14.124 + evaluator_(evaluator),
14.125 + current_(start ? start->first() : NULL)
14.126 + {
14.127 + }
14.128 +
14.129 + Object *next() {
14.130 + while (current_ && current_ != top_) {
14.131 + Object *obj = reinterpret_cast<Object *>(current_);
14.132 + size_t size = evaluator_(obj, obj->map());
14.133 + current_ += size;
14.134 + if (current_ == chunk_->end()) {
14.135 + chunk_ = chunk_->next();
14.136 + current_ = chunk_ ? chunk_->first() : NULL;
14.137 + }
14.138 + if (obj->is(MapKind_FreeRegion))
14.139 + continue;
14.140 + }
14.141 + return NULL;
14.142 + }
14.143 };
14.144
14.145 -class ChunkObjectIterator
14.146 +class FreedObject
14.147 {
14.148 - HeapChunk *start_;
14.149 - Address top_;
14.150 + Map *map_;
14.151 + size_t bytes_;
14.152 + FreedObject *next_;
14.153 +
14.154 + public:
14.155 + FreedObject(Map *map, size_t bytes)
14.156 + : map_(map),
14.157 + bytes_(bytes)
14.158 + {
14.159 + assert(bytes_ >= sizeof(Map *) + sizeof(size_t));
14.160 + if (bytes_ >= sizeof(FreedObject))
14.161 + next_ = NULL;
14.162 + }
14.163 +
14.164 + void setNext(FreedObject *obj) {
14.165 + assert(bytes_ >= sizeof(FreedObject));
14.166 + next_ = obj;
14.167 + }
14.168 + FreedObject *next() {
14.169 + assert(bytes_ >= sizeof(FreedObject));
14.170 + return next_;
14.171 + }
14.172 + size_t size() {
14.173 + return bytes_;
14.174 + }
14.175 +};
14.176 +
14.177 +// Free-region list used in old spaces. Based on V8's. Each object pool
14.178 +// is used to allocate objects of the previous pool's size. For example,
14.179 +// the medium object pool is used to allocate small objects.
14.180 +class FreeList
14.181 +{
14.182 + private:
14.183 + static const size_t kMinObjectSize = 32;
14.184 + static const size_t kMaxSmallObject = 255;
14.185 + static const size_t kMaxMediumObject = (2 * kKB) - 1;
14.186 + static const size_t kMaxLargeObject = (16 * kKB) - 1;
14.187 +
14.188 + public:
14.189 + FreeList();
14.190 +
14.191 + public:
14.192 + void add(FreedObject *obj);
14.193 + FreedObject *pick(size_t size);
14.194 + void reset();
14.195 +
14.196 + private:
14.197 + FreedObject *small_objects_;
14.198 + FreedObject *medium_objects_;
14.199 + FreedObject *large_objects_;
14.200 + FreedObject *huge_objects_;
14.201 };
14.202
14.203 // The heap stack is designed for structures which are known to not escape the
14.204 @@ -253,16 +371,23 @@
14.205 inline bool contains(Object *obj) {
14.206 return (reinterpret_cast<uintptr_t>(obj) & mask_) == reinterpret_cast<uintptr_t>(chunk_);
14.207 }
14.208 +
14.209 + template <typename Evaluator>
14.210 + ChunkObjectIteratorBase<Evaluator> begin(const Evaluator &evaluator) {
14.211 + return ChunkObjectIteratorBase<Evaluator>(chunk_, cursor_, evaluator);
14.212 + }
14.213 };
14.214
14.215 class OldSpace
14.216 {
14.217 + Zone *zone_;
14.218 HeapChunk *first_;
14.219 HeapChunk *last_;
14.220 Address cursor_;
14.221 + FreeList freelist_;
14.222
14.223 public:
14.224 - OldSpace();
14.225 + OldSpace(Zone *zone);
14.226 ~OldSpace();
14.227
14.228 bool initialize();
14.229 @@ -270,6 +395,18 @@
14.230 public:
14.231 inline Address allocate(size_t nbytes);
14.232 Address allocateSlow(size_t nbytes);
14.233 + void markRegionFree(Address start, size_t size);
14.234 + void freeUnusedPages();
14.235 +
14.236 + template <typename Evaluator>
14.237 + ChunkObjectIteratorBase<Evaluator> begin(const Evaluator &evaluator) {
14.238 + return ChunkObjectIteratorBase<Evaluator>(first_, cursor_, evaluator);
14.239 + }
14.240 +
14.241 + HeapChunk *first() {
14.242 + return first_;
14.243 + }
14.244 + void resetFreeList();
14.245 };
14.246
14.247 class LargeObjectSpace
14.248 @@ -285,6 +422,11 @@
14.249
14.250 public:
14.251 Address allocate(size_t nbytes);
14.252 +
14.253 + HeapChunk *first() {
14.254 + return first_;
14.255 + }
14.256 + void sweep();
14.257 };
14.258
14.259 }
15.1 --- a/jitcraft/Types.cpp Sat Jul 07 01:45:33 2012 -0400
15.2 +++ b/jitcraft/Types.cpp Sat Jul 07 17:11:31 2012 -0700
15.3 @@ -31,6 +31,8 @@
15.4 type->kind_ = PRIMITIVE;
15.5 type->tag_ = tag;
15.6 type->contained_ = NULL;
15.7 + type->returnType_ = NULL;
15.8 + type->parameters_ = NULL;
15.9
15.10 if (tag && strcmp(tag, "Float") == 0)
15.11 type->primitive_ = PrimitiveType_Float;
15.12 @@ -51,6 +53,8 @@
15.13 type->kind_ = PRIMITIVE;
15.14 type->tag_ = NULL;
15.15 type->contained_ = NULL;
15.16 + type->returnType_ = NULL;
15.17 + type->parameters_ = NULL;
15.18 type->primitive_ = prim;
15.19 return type;
15.20 }
15.21 @@ -65,6 +69,8 @@
15.22 type->kind_ = ARRAY;
15.23 type->tag_ = NULL;
15.24 type->contained_ = contained;
15.25 + type->returnType_ = NULL;
15.26 + type->parameters_ = NULL;
15.27 type->elements_ = elements;
15.28 if (contained->isPrimitive())
15.29 type->levels_ = 1;
15.30 @@ -84,6 +90,8 @@
15.31 type->kind_ = REFERENCE;
15.32 type->tag_ = NULL;
15.33 type->contained_ = contained;
15.34 + type->returnType_ = NULL;
15.35 + type->parameters_ = NULL;
15.36 return type;
15.37 }
15.38
15.39 @@ -95,6 +103,7 @@
15.40 return NULL;
15.41
15.42 type->kind_ = FUNCTION;
15.43 + type->contained_ = NULL;
15.44 type->returnType_ = returnType;
15.45 type->parameters_ = parameters;
15.46 return type;
16.1 --- a/jitcraft/Utility.h Sat Jul 07 01:45:33 2012 -0400
16.2 +++ b/jitcraft/Utility.h Sat Jul 07 17:11:31 2012 -0700
16.3 @@ -75,7 +75,7 @@
16.4 {
16.5 assert(number != 0);
16.6
16.7 -#ifdef MSC_VER
16.8 +#ifdef _MSC_VER
16.9 unsigned long rval;
16.10 _BitScanReverse(&rval, number);
16.11 return rval;
16.12 @@ -88,6 +88,24 @@
16.13 #endif
16.14 }
16.15
16.16 +static inline size_t
16.17 +FindRightmostBit(size_t number)
16.18 +{
16.19 + assert(number != 0);
16.20 +
16.21 +#ifdef _MSC_VER
16.22 + unsigned long rval;
16.23 + _BitScanForward(&rval, number);
16.24 + return rval;
16.25 +#else
16.26 + size_t bit;
16.27 + asm("bsf %1, %0\n"
16.28 + : "=r" (bit)
16.29 + : "rm" (number));
16.30 + return bit;
16.31 +#endif
16.32 +}
16.33 +
16.34 static inline bool
16.35 IsPowerOfTwo(size_t value)
16.36 {
17.1 --- a/jitcraft/Zone.cpp Sat Jul 07 01:45:33 2012 -0400
17.2 +++ b/jitcraft/Zone.cpp Sat Jul 07 17:11:31 2012 -0700
17.3 @@ -55,6 +55,8 @@
17.4 return false;
17.5 if ((maps_[MapKind_Function] = allocateMap(MapKind_Function, sizeof(Map))) == NULL)
17.6 return false;
17.7 + if ((maps_[MapKind_FreeRegion] = allocateMap(MapKind_FreeRegion, sizeof(Map))) == NULL)
17.8 + return false;
17.9
17.10 types_ = new TypeManager(this);
17.11 if (!types_ || !types_->initialize())
18.1 --- a/jitcraft/Zone.h Sat Jul 07 01:45:33 2012 -0400
18.2 +++ b/jitcraft/Zone.h Sat Jul 07 17:11:31 2012 -0700
18.3 @@ -61,12 +61,15 @@
18.4 }
18.5 Object *allocate(MapKind kind, size_t bytes, Heap::Tenure tenure = Heap::Tenure_Default) {
18.6 assert(maps_[kind]);
18.7 - Local<Map> map(this, maps_[kind]);
18.8 + Local<Map> map(this, maps_[kind]);
18.9 return allocate(map, bytes, tenure);
18.10 }
18.11 Object *allocate(Handle<Map> map, size_t bytes, Heap::Tenure tenure = Heap::Tenure_Default) {
18.12 return heap_.allocate(map, bytes, tenure);
18.13 }
18.14 + Map *getMap(MapKind kind) {
18.15 + return maps_[kind];
18.16 + }
18.17 Map *allocateMap(MapKind kind, size_t bytes, MapKind parent = MapKind_Meta);
18.18 TypeManager *types() const {
18.19 return types_;
18.20 @@ -130,6 +133,12 @@
18.21 }
18.22
18.23 template <typename T>
18.24 +ScopedRoot<T>::ScopedRoot()
18.25 + : thingp_(make(NULL))
18.26 +{
18.27 +}
18.28 +
18.29 +template <typename T>
18.30 ScopedRoot<T>::ScopedRoot(T *thing)
18.31 : thingp_(make(thing))
18.32 {
19.1 --- a/jitcraft/jitcraft.vcxproj Sat Jul 07 01:45:33 2012 -0400
19.2 +++ b/jitcraft/jitcraft.vcxproj Sat Jul 07 17:11:31 2012 -0700
19.3 @@ -54,6 +54,7 @@
19.4 <Link>
19.5 <SubSystem>Console</SubSystem>
19.6 <GenerateDebugInformation>true</GenerateDebugInformation>
19.7 + <RandomizedBaseAddress>false</RandomizedBaseAddress>
19.8 </Link>
19.9 </ItemDefinitionGroup>
19.10 <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
19.11 @@ -91,6 +92,7 @@
19.12 <ClCompile Include="Platform-Windows.cpp" />
19.13 <ClCompile Include="PoolAllocator.cpp" />
19.14 <ClCompile Include="Reference.cpp" />
19.15 + <ClCompile Include="RootScopes.cpp" />
19.16 <ClCompile Include="Scanner.cpp" />
19.17 <ClCompile Include="Scopes.cpp" />
19.18 <ClCompile Include="SemanticAnalysis.cpp" />
19.19 @@ -117,6 +119,8 @@
19.20 <ClInclude Include="Modules.h" />
19.21 <ClInclude Include="NameBinding.h" />
19.22 <ClInclude Include="Heap.h" />
19.23 + <ClInclude Include="Objects.h" />
19.24 + <ClInclude Include="ObjectVisitor.h" />
19.25 <ClInclude Include="Opcodes.h" />
19.26 <ClInclude Include="Parser.h" />
19.27 <ClInclude Include="Platform.h" />