/* vim: set ts=4 sw=4 tw=99 et:

 *

 * Copyright (C) 2012 David Anderson

 *

 * This file is part of JITCraft.

 *

 * JITCraft is free software: you can redistribute it and/or modify it under

 * the terms of the GNU General Public License as published by the Free

 * Software Foundation, either version 3 of the License, or (at your option)

 * any later version.

 * 

 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY

 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License along with

 * JITCraft. If not, see http://www.gnu.org/licenses/.

 */

#include "Heap.h"

#include "Heap-inl.h"

#include "Spaces-inl.h"

#include "VirtualObjectVisitor.h"

#include "ObjectVisitor.h"

#include "Zone.h"

#include "StringPool.h"

using namespace jc;

static const size_t kMarkStackSize = 256 * kKB;

Heap::Heap(Zone *zone)

  : zone_(zone),

    markStack_(NULL),

    oldSpace_(zone),

    mapSpace_(zone)

{

}

bool

Heap::initialize()

{

    if (!lifoSpace_.initialize())

        return false;

    if (!oldSpace_.initialize())

        return false;

    if (!mapSpace_.initialize())

        return false;

    if (!largeSpace_.initialize())

        return false;

    markStack_ = malloc(kMarkStackSize);

    if (!markStack_)

        return false;

    if ((metaMap_ = allocateMap(MapKind_Meta, sizeof(Map))) == NULL)

        return false;

    metaMap_->init(metaMap_);

    return true;

}

Heap::~Heap()

{

    free(markStack_);

}

Map *

Heap::allocateMap(MapKind kind, size_t bytes)

{

    size_t actualBytes = Align(bytes, kObjectAlignment);

    Address addr = mapSpace_.allocate(actualBytes);

    if (!addr) {

        assert(false);

        return NULL;

    }

    // Note that we use a NULL type initially, and reinterpret-cast to get

    // the Type pointer, so we can allocate the self-referencing meta type.

    Object *obj = InitAddressIntoObject(addr, NULL);

    Map *map = reinterpret_cast<Map *>(obj);

    map->setKind(kind);

    return map;

}

// To avoid marking recursively, we use a worklist to manage the marking

// process. Algorithm:

//

// (1) While the mark stack is not empty:

//  (a) Remove object |o| from the mark stack.

//  (b) For each reference |p| in object |o|,

//   (i)  Mark |p| as Black.

//   (ii) Push |p| onto the mark stack.

//   (iv) If the mark stack overflowde, mark |p| as Grey.

//

// We run this algorithm after pushing every root object onto the mark stack.

// After all roots have been processed, it may be that the mark stack

// overflowed. In this case, we use the following algorithm:

//

//  (1) while the mark stack has overflowed:

//   (a) Search the heap for grey objects, adding as many as possible to the

//       mark stack.

//   (b) Run the mark-stack-emptying algorithm above.

//

// This algorithm is guaranteed to terminate, because even if there are more

// grey objects than mark stack capacity, we eventually all objects will

// become grey, and grey objects are not re-pushed onto the mark stack. Thus,

// eventually, all objects will become black.

//

namespace jc {

class MarkStack

{

    Object **base_;

    Object **top_;

    Object **end_;

    bool overflowed_;

  public:

    MarkStack(Object **base, Object **end)

      : base_(base),

        top_(base),

        end_(end),

        overflowed_(false)

    {

    }

    void pushBlack(Object *ptr) {

        assert(HeapChunk::IsBlack(HeapChunk::GetMarkBit(ptr)));

        if (top_ >= end_) {

            MarkBit bit = HeapChunk::GetMarkBit(ptr);

            HeapChunk::BlackToGrey(bit);

            overflowed_ = true;

            return;

        }

        *top_++ = ptr;

    }

    Object *pop() {

        assert(!empty());

        return *--top_;

    }

    bool overflowed() const {

        return overflowed_;

    }

    bool empty() const {

        return base_ == top_;

    }

    void reset() {

        assert(empty());

        overflowed_ = true;

    }

};

} // namespace jc

// Iterates over an object's body, pushing its referenced objects onto the

// mark stack.

class MarkingVisitor : public ObjectVisitor<VirtualObjectVisitor>

{

    MarkStack &stack_;

  public:

    MarkingVisitor(MarkStack &stack)

      : stack_(stack)

    {

    }

    void visit(Object **ptr) {

        if (!*ptr)

            return;

        Object *obj = *ptr;

        // If the object is grey, we'll find it later when searching the heap.

        // If the object is black, it's already been scsanned.

        MarkBit bit = HeapChunk::GetMarkBit(obj);

        if (HeapChunk::IsBlackOrGrey(bit))

            return;

        // Place the object in the stack and optimistically mark it black. If

        // the stack overflows, it'll become grey instead.

        HeapChunk::WhiteToBlack(bit);

        stack_.pushBlack(obj);

    }

};

static void

ProcessMarkStack(MarkStack &stack)

{

    MarkingVisitor visitor(stack);

    while (!stack.empty()) {

        Object *obj = stack.pop();

        assert(HeapChunk::IsBlack(HeapChunk::GetMarkBit(obj)));

        visitor.iterate(obj);

    }

}

class RootMarkingVisitor : public VirtualObjectVisitor

{

    MarkStack &stack_;

    MarkingVisitor visitor_;

  public:

    RootMarkingVisitor(MarkStack &stack)

      : stack_(stack),

        visitor_(stack)

    {

    }

    void visit(Object **ptr) {

        // Flush the stack after every new root. Note that the root gets pushed

        // as black; that's okay. If the mark stack overflows, its children are

        // guaranteed to be at least grey.

        visitor_.visit(ptr);

        ProcessMarkStack(stack_);

    }

};

typedef ChunkObjectIteratorBase<ObjectSizeEvaluator> ChunkObjectIterator;

static inline bool

EnqueueGreyObject(MarkStack &stack, Object *obj)

{

    assert(!stack.overflowed());

    MarkBit bit = HeapChunk::GetMarkBit(obj);

    if (HeapChunk::IsGrey(bit)) {

        HeapChunk::GreyToBlack(bit);

        stack.pushBlack(obj);

        return !stack.overflowed();

    }

    return true;

}

static void

EnqueueGreyObjectsInChunks(MarkStack &stack, ChunkObjectIterator iter)

{

    for (Object *obj = iter.next(); obj; obj = iter.next()) {

        if (!EnqueueGreyObject(stack, obj))

            return;

    }

}

static void

EnqueueGreyObjectsInLargeChunks(MarkStack &stack, HeapChunk *first)

{

    for (HeapChunk *chunk = first; chunk; chunk = chunk->next()) {

        Object *obj = reinterpret_cast<Object *>(chunk->first());

        if (!EnqueueGreyObject(stack, obj))

            return;

    }

}

void

Heap::findGreyObjects(MarkStack &stack)

{

    EnqueueGreyObjectsInChunks(stack, lifoSpace_.begin(ObjectSizeEvaluator()));

    if (stack.overflowed())

        return;

    EnqueueGreyObjectsInChunks(stack, mapSpace_.begin(ObjectSizeEvaluator()));

    if (stack.overflowed())

        return;

    EnqueueGreyObjectsInChunks(stack, oldSpace_.begin(ObjectSizeEvaluator()));

    if (stack.overflowed())

        return;

    EnqueueGreyObjectsInLargeChunks(stack, largeSpace_.first());

    if (stack.overflowed())

        return;

}

void

Heap::mark()

{

    Object **qbase = reinterpret_cast<Object **>(markStack_);

    Object **qend = qbase + (kMarkStackSize / sizeof(Object *));

    MarkStack stack(qbase, qend);

    RootMarkingVisitor visitor(stack);

    // Attempt to mark all objects reachable from the root set.

    zone_->accept(&visitor);

    assert(stack.empty());

    // If the stack overflowed, we need to look through the entire heap for

    // grey objects to rescan. We keep doing this until there are no grey

    // objects left.

    while (stack.overflowed()) {

        stack.reset();

        findGreyObjects(stack);

        ProcessMarkStack(stack);

    }

}

// Use the mark bits to scan for live objects, and mark the regions in

// between them as free.

static void

SweepOldSpaceChunk(OldSpace *space, HeapChunk *chunk)

{

    uint32 *markbits = chunk->markBits();

    uint32 *markbitsEnd = markbits + HeapChunk::kMarkWords;

    Address freeStart = chunk->first();

    // Scoot up to the first usable markword. Since memory is reserved for

    // mark bits, not all addresses are allocatable, so quite a few of the

    // initial mark bits are wasted. (1KB of mark bytes on x86.)

    markbits = HeapChunk::GetMarkBit(reinterpret_cast<Object *>(chunk->first())).word();

    uint32 bits = *markbits;

    for (;;) {

        while (bits) {

            uint32 bit = FindRightmostBit(bits);

            Address object = HeapChunk::FromMarkBit(chunk, markbits, bit);

            if (freeStart != object)

                space->markRegionFree(freeStart, object - freeStart);

            Object *obj = reinterpret_cast<Object *>(object);

            freeStart = object + SizeOfObject(obj, obj->map());

            bits &= ~(1 << bit);

            if (bits == HeapChunk::kMarkBitsPerWord - 1) {

                // This bit is the last bit on this word, so bump up to the

                // next word and exclude the first bit.

                markbits++;

                assert(markbits < markbitsEnd);

                bits = *markbits & ~(1 << 0);

            }

        }

        markbits++;

        bits = *markbits;

        if (markbits >= markbitsEnd)

            break;

    }

    if (freeStart == chunk->base())

        chunk->setEmpty();

    else if (freeStart != chunk->end())

        space->markRegionFree(freeStart, chunk->end() - freeStart);

}

static void

SweepOldSpace(OldSpace *space)

{

    space->resetFreeList();

    for (HeapChunk *chunk = space->first(); chunk; chunk = chunk->next()) {

        SweepOldSpaceChunk(space, chunk);

        chunk->resetMarkBits();

    }

    space->freeUnusedPages();

}

void

Heap::sweep()

{

    // Before destroying mark bits, sweep weak roots.

    zone_->symbols()->sweep();

    // Note: we don't bother sweeping the LIFO space.

    SweepOldSpace(&mapSpace_);

    SweepOldSpace(&oldSpace_);

    largeSpace_.sweep();

}

void

Heap::reclaim()

{

    mark();

    sweep();

}