diff --git a/bin/GCStress/GCStress.cpp b/bin/GCStress/GCStress.cpp index bf6219dbe71..3c6be01cfc9 100644 --- a/bin/GCStress/GCStress.cpp +++ b/bin/GCStress/GCStress.cpp @@ -214,16 +214,23 @@ void BuildObjectCreationTable() objectCreationTable.AddWeightedEntry(&ScannedObject<1, 50>::New, 10000); objectCreationTable.AddWeightedEntry(&BarrierObject<1, 50>::New, 2000); objectCreationTable.AddWeightedEntry(&TrackedObject<1, 50>::New, 2000); +#ifdef RECYCLER_VISITED_HOST + objectCreationTable.AddWeightedEntry(&RecyclerVisitedObject<1, 50>::New, 2000); +#endif objectCreationTable.AddWeightedEntry(&LeafObject<51, 1000>::New, 10); objectCreationTable.AddWeightedEntry(&ScannedObject<51, 1000>::New, 100); objectCreationTable.AddWeightedEntry(&BarrierObject<51, 1000>::New, 20); objectCreationTable.AddWeightedEntry(&TrackedObject<51, 1000>::New, 20); +#ifdef RECYCLER_VISITED_HOST + objectCreationTable.AddWeightedEntry(&RecyclerVisitedObject<51, 1000>::New, 40); +#endif objectCreationTable.AddWeightedEntry(&LeafObject<1001, 50000>::New, 1); objectCreationTable.AddWeightedEntry(&ScannedObject<1001, 50000>::New, 10); objectCreationTable.AddWeightedEntry(&BarrierObject<1001, 50000>::New, 2); // objectCreationTable.AddWeightedEntry(&TrackedObject<1001, 50000>::New, 2); // Large tracked objects are not supported +// objectCreationTable.AddWeightedEntry(&RecyclerVisitedObject<1001, 50000>::New, 2); // Large recycler visited objects are not supported } void BuildOperationTable() diff --git a/bin/GCStress/RecyclerTestObject.cpp b/bin/GCStress/RecyclerTestObject.cpp index fbf9f442bb7..52280efc047 100644 --- a/bin/GCStress/RecyclerTestObject.cpp +++ b/bin/GCStress/RecyclerTestObject.cpp @@ -10,6 +10,7 @@ size_t RecyclerTestObject::walkObjectCount = 0; size_t RecyclerTestObject::walkScannedByteCount = 0; size_t RecyclerTestObject::walkBarrierByteCount = 0; size_t RecyclerTestObject::walkTrackedByteCount = 0; +size_t RecyclerTestObject::walkRecyclerVisitedByteCount = 0; size_t RecyclerTestObject::walkLeafByteCount = 0; size_t RecyclerTestObject::currentWalkDepth = 0; size_t RecyclerTestObject::maxWalkDepth = 0; diff --git a/bin/GCStress/RecyclerTestObject.h b/bin/GCStress/RecyclerTestObject.h index 1f0004ad2c1..f57379b9004 100644 --- a/bin/GCStress/RecyclerTestObject.h +++ b/bin/GCStress/RecyclerTestObject.h @@ -4,7 +4,9 @@ //------------------------------------------------------------------------------------------------------- #include "stdafx.h" -class RecyclerTestObject : public FinalizableObject +#include "Core/RecyclerHeapMarkingContext.h" + +class RecyclerTestObject : public IRecyclerVisitedObject { protected: RecyclerTestObject() @@ -13,10 +15,16 @@ class RecyclerTestObject : public FinalizableObject } public: - // FinalizableObject implementation + // IRecyclerVisitedObject implementation. We don't use FinalizableObject here as + // RecyclerVisitedObjects need to have Trace called on them, which is not allowed for + // FinalizableObject. virtual void Finalize(bool isShutdown) override { VerifyCondition(false); }; virtual void Dispose(bool isShutdown) override { VerifyCondition(false); }; - virtual void Mark(Recycler * recycler) override { VerifyCondition(false); }; + virtual void OnMark() override {} + virtual void Mark(RecyclerHeapHandle recycler) override { Mark(static_cast(recycler)); }; + virtual void Trace(IRecyclerHeapMarkingContext* markContext) override { VerifyCondition(false); }; + + virtual void Mark(Recycler * recycler) { VerifyCondition(false); }; public: static void BeginWalk() @@ -27,6 +35,7 @@ class RecyclerTestObject : public FinalizableObject walkScannedByteCount = 0; walkBarrierByteCount = 0; walkTrackedByteCount = 0; + walkRecyclerVisitedByteCount = 0; walkLeafByteCount = 0; maxWalkDepth = 0; @@ -66,13 +75,14 @@ class RecyclerTestObject : public FinalizableObject VerifyCondition(currentWalkDepth == 0); wprintf(_u("Full heap walk finished\n")); - wprintf(_u("Object Count: %12llu\n"), (unsigned long long) walkObjectCount); - wprintf(_u("Scanned Bytes: %12llu\n"), (unsigned long long) walkScannedByteCount); - wprintf(_u("Barrier Bytes: %12llu\n"), (unsigned long long) walkBarrierByteCount); - wprintf(_u("Tracked Bytes: %12llu\n"), (unsigned long long) walkTrackedByteCount); - wprintf(_u("Leaf Bytes: %12llu\n"), (unsigned long long) walkLeafByteCount); - wprintf(_u("Total Bytes: %12llu\n"), (unsigned long long) (walkScannedByteCount + walkBarrierByteCount + walkTrackedByteCount + walkLeafByteCount)); - wprintf(_u("Max Depth: %12llu\n"), (unsigned long long) maxWalkDepth); + wprintf(_u("Object Count: %12llu\n"), (unsigned long long) walkObjectCount); + wprintf(_u("Scanned Bytes: %12llu\n"), (unsigned long long) walkScannedByteCount); + wprintf(_u("Barrier Bytes: %12llu\n"), (unsigned long long) walkBarrierByteCount); + wprintf(_u("Tracked Bytes: %12llu\n"), (unsigned long long) walkTrackedByteCount); + wprintf(_u("RecyclerVisited Bytes: %12llu\n"), (unsigned long long) walkRecyclerVisitedByteCount); + wprintf(_u("Leaf Bytes: %12llu\n"), (unsigned long long) walkLeafByteCount); + wprintf(_u("Total Bytes: %12llu\n"), (unsigned long long) (walkScannedByteCount + walkBarrierByteCount + walkTrackedByteCount + walkLeafByteCount + walkRecyclerVisitedByteCount)); + wprintf(_u("Max Depth: %12llu\n"), (unsigned long long) maxWalkDepth); } // Virtual methods @@ -100,6 +110,7 @@ class RecyclerTestObject : public FinalizableObject static size_t walkLeafByteCount; static size_t walkBarrierByteCount; static size_t walkTrackedByteCount; + static size_t walkRecyclerVisitedByteCount; static size_t currentWalkDepth; static size_t maxWalkDepth; @@ -232,8 +243,13 @@ class BarrierObject : public RecyclerTestObject FieldNoBarrier(RecyclerTestObject *) references[0]; // SWB-TODO: is this correct? }; +// TrackedObject must be a FinalizableObject (in order to be 'new'ed with RecyclerNewTrackedLeafPlusZ) +// but it also must be a RecyclerTestObject to participate in GCStress. It must inherit from RecyclerTestObject +// first so that the algined pointer is returned from New. +// Fortunately, the v-tables for RecyclerTestObject and FinalizableObject line up, so the +// IRecyclerVisitedObject/FinalizableObject calls end up in the right place. template -class TrackedObject : public RecyclerTestObject +class TrackedObject : public RecyclerTestObject, public FinalizableObject { private: TrackedObject(unsigned int count) : @@ -295,4 +311,124 @@ class TrackedObject : public RecyclerTestObject FieldNoBarrier(RecyclerTestObject *) references[0]; // SWB-TODO: is this correct? }; +#ifdef RECYCLER_VISITED_HOST + +template +class RecyclerVisitedObject : public RecyclerTestObject +{ +public: + static RecyclerTestObject * New() + { + // Determine a random amount of RecyclerTestObject* references to influence the size of this object. + const unsigned int count = minCount + GetRandomInteger(maxCount - minCount + 1); + + void* mem = nullptr; + const size_t size = sizeof(RecyclerVisitedObject) + (sizeof(RecyclerTestObject*) * count); + + // Randomly select the type of object to create + AllocationType allocType = static_cast(GetRandomInteger(static_cast(AllocationType::Count))); + switch (allocType) + { + case AllocationType::TraceAndFinalized: + mem = RecyclerAllocVisitedHostTracedAndFinalizedZero(recyclerInstance, size); + break; + case AllocationType::TraceOnly: + mem = RecyclerAllocVisitedHostTracedZero(recyclerInstance, size); + break; + case AllocationType::FinalizeLeaf: + mem = RecyclerAllocVisitedHostFinalizedZero(recyclerInstance, size); + break; + default: + Assert(allocType == AllocationType::Leaf); + mem = RecyclerAllocLeafZero(recyclerInstance, size); + } + + // Construct the v-table, allocType, and count information for the new object. + RecyclerVisitedObject* obj = new (mem) RecyclerVisitedObject(allocType, count); + return obj; + } + + virtual bool TryGetRandomLocation(Location * location) override + { + // Leaf types should not return a location + if (type == AllocationType::Leaf || type == AllocationType::FinalizeLeaf) + { + return false; + } + // Get a random slot and construct a Location for it + // Make this a Tagged location so that we won't inadvertently keep objects alive + // in the case where this object gets put on the wrong mark stack. + *location = Location::Tagged(&references[GetRandomInteger(count)]); + + return true; + } + + virtual void Trace(IRecyclerHeapMarkingContext* markContext) override + { + VerifyCondition(type == AllocationType::TraceAndFinalized || type == AllocationType::TraceOnly); + // Note that the pointers in the references arrary are technically tagged. However, this is ok + // as the Mark that we're performing is an interior mark, which gets us to the right object(s). + markContext->MarkObjects(reinterpret_cast(&references[0]), count, this); + } + + virtual void Finalize(bool isShutdown) override + { + // Only types that request finalization should have Finalize called + VerifyCondition(IsFinalizable()); + } + virtual void Dispose(bool isShutdown) override + { + // Only types that request finalization should have Finalize called + VerifyCondition(IsFinalizable()); + VerifyCondition(unmanagedResource != nullptr); + BOOL success = ::HeapFree(GetProcessHeap(), 0, unmanagedResource); + VerifyCondition(success != FALSE); + unmanagedResource = nullptr; + } + + +protected: + virtual void DoWalkObject() override + { + walkRecyclerVisitedByteCount += sizeof(RecyclerVisitedObject) + count * sizeof(RecyclerTestObject *); + + for (unsigned int i = 0; i < count; i++) + { + RecyclerTestObject::WalkReference(Location::Untag(references[i])); + } + } + +private: + enum class AllocationType : unsigned int + { + TraceAndFinalized = 0, + TraceOnly, + FinalizeLeaf, + Leaf, + Count, + }; + + bool IsFinalizable() const { return type == AllocationType::TraceAndFinalized || type == AllocationType::FinalizeLeaf; } + RecyclerVisitedObject(AllocationType allocType, unsigned int count) : + count(count), + type(allocType) + { + for (unsigned int i = 0; i < count; i++) + { + references[i] = nullptr; + } + if (IsFinalizable()) + { + unmanagedResource = ::HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, GetRandomInteger(1024)); + VerifyCondition(unmanagedResource != nullptr); + } + } + + + Field(AllocationType) type; + Field(void*) unmanagedResource; + Field(unsigned int) count; + FieldNoBarrier(RecyclerTestObject *) references[0]; // SWB-TODO: is this correct? (copied from TrackedObject) +}; +#endif diff --git a/bin/ch/Helpers.cpp b/bin/ch/Helpers.cpp index 1b5108a3a07..6289ecc65b3 100644 --- a/bin/ch/Helpers.cpp +++ b/bin/ch/Helpers.cpp @@ -191,6 +191,9 @@ HRESULT Helpers::LoadScriptFromFile(LPCSTR filename, LPCSTR& contents, UINT* len // wrongly classified as ANSI // { +#pragma warning(push) +// suppressing prefast warning that "readable size is bufferLength bytes but 2 may be read" as bufferLength is clearly > 2 in the code that follows +#pragma warning(disable:6385) C_ASSERT(sizeof(WCHAR) == 2); if (bufferLength > 2) { @@ -211,6 +214,7 @@ HRESULT Helpers::LoadScriptFromFile(LPCSTR filename, LPCSTR& contents, UINT* len #pragma prefast(pop) } } +#pragma warning(pop) } contents = reinterpret_cast(pRawBytes); diff --git a/lib/Common/CommonDefines.h b/lib/Common/CommonDefines.h index d5178b085e8..11def475a0a 100644 --- a/lib/Common/CommonDefines.h +++ b/lib/Common/CommonDefines.h @@ -239,6 +239,10 @@ #error "Background page zeroing can't be turned on if freeing pages in the background is disabled" #endif +#ifdef _WIN32 +#define RECYCLER_VISITED_HOST +#endif + // JIT features #if DISABLE_JIT diff --git a/lib/Common/CommonMinMemory.h b/lib/Common/CommonMinMemory.h index 3c9c40597d5..b971cfe7810 100644 --- a/lib/Common/CommonMinMemory.h +++ b/lib/Common/CommonMinMemory.h @@ -40,5 +40,6 @@ class FinalizableObject; #include "Memory/RecyclerSweep.h" #include "Memory/RecyclerHeuristic.h" #include "Memory/MarkContext.h" +#include "Memory/MarkContextWrapper.h" #include "Memory/RecyclerWatsonTelemetry.h" #include "Memory/Recycler.h" diff --git a/lib/Common/Core/FinalizableObject.h b/lib/Common/Core/FinalizableObject.h index 30088930174..60091395079 100644 --- a/lib/Common/Core/FinalizableObject.h +++ b/lib/Common/Core/FinalizableObject.h @@ -3,24 +3,22 @@ // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. //------------------------------------------------------------------------------------------------------- #pragma once -class FinalizableObject +#include "RecyclerVisitedObject.h" + +class FinalizableObject : public IRecyclerVisitedObject { public: - // Called right after finish marking and this object is determined to be dead. - // Should contain only simple clean up code. - // Can't run another script - // Can't cause a re-entrant collection - - virtual void Finalize(bool isShutdown) = 0; - - // Call after sweeping is done. - // Can call other script or cause another collection. + virtual void OnMark() {} - virtual void Dispose(bool isShutdown) = 0; + void Mark(RecyclerHeapHandle recycler) final + { + Mark(static_cast(recycler)); + } - // Used only by TrackableObjects (created with TrackedBit on by RecyclerNew*Tracked) - virtual void Mark(Recycler * recycler) = 0; + void Trace(IRecyclerHeapMarkingContext* markingContext) final + { + AssertMsg(false, "Trace called on object that isn't implemented by the host"); + } - // Special behavior on certain GC's - virtual void OnMark() {} + virtual void Mark(Recycler* recycler) = 0; }; diff --git a/lib/Common/Core/RecyclerHeapMarkingContext.h b/lib/Common/Core/RecyclerHeapMarkingContext.h new file mode 100644 index 00000000000..719cff179bb --- /dev/null +++ b/lib/Common/Core/RecyclerHeapMarkingContext.h @@ -0,0 +1,11 @@ +//------------------------------------------------------------------------------------------------------- +// Copyright (C) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. +//------------------------------------------------------------------------------------------------------- +#pragma once + +interface IRecyclerHeapMarkingContext +{ + virtual void MarkObjects(void** objects, size_t count, void* parent) = 0; +}; + diff --git a/lib/Common/Core/RecyclerVisitedObject.h b/lib/Common/Core/RecyclerVisitedObject.h new file mode 100644 index 00000000000..ca1c50e1d75 --- /dev/null +++ b/lib/Common/Core/RecyclerVisitedObject.h @@ -0,0 +1,31 @@ +//------------------------------------------------------------------------------------------------------- +// Copyright (C) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. +//------------------------------------------------------------------------------------------------------- +#pragma once + +interface IRecyclerHeapMarkingContext; +typedef void* RecyclerHeapHandle; + +interface IRecyclerVisitedObject +{ + // Called right after finish marking and this object is determined to be dead. + // Should contain only simple clean up code. + // Can't run another script + // Can't cause a re-entrant collection + virtual void Finalize(bool isShutdown) = 0; + + // Call after sweeping is done. + // Can call other script or cause another collection. + virtual void Dispose(bool isShutdown) = 0; + + // Used only by TrackableObjects (created with TrackedBit on by RecyclerNew*Tracked) + virtual void Mark(RecyclerHeapHandle recycler) = 0; + + // Special behavior on certain GC's + virtual void OnMark() = 0; + + // Used only by RecyclerVisitedHost objects (created with RecyclerAllocVistedHost_Traced*) + virtual void Trace(IRecyclerHeapMarkingContext* markingContext) = 0; +}; + diff --git a/lib/Common/Exceptions/ReportError.h b/lib/Common/Exceptions/ReportError.h index 0f2f4922b53..31e0931b382 100644 --- a/lib/Common/Exceptions/ReportError.h +++ b/lib/Common/Exceptions/ReportError.h @@ -27,6 +27,7 @@ enum ErrorReason Fatal_JsReentrancy_Error = 19, Fatal_TTDAbort = 20, Fatal_Failed_API_Result = 21, + Fatal_RecyclerVisitedHost_LargeHeapBlock = 22, }; extern "C" void ReportFatalException( diff --git a/lib/Common/Memory/HeapBlock.cpp b/lib/Common/Memory/HeapBlock.cpp index 551373f2c87..a76afe86686 100644 --- a/lib/Common/Memory/HeapBlock.cpp +++ b/lib/Common/Memory/HeapBlock.cpp @@ -31,6 +31,16 @@ HeapBlock::AsFinalizableBlock() return static_cast *>(this); } +#ifdef RECYCLER_VISITED_HOST +template +SmallRecyclerVisitedHostHeapBlockT * +HeapBlock::AsRecyclerVisitedHostBlock() +{ + Assert(IsRecyclerVisitedHostBlock()); + return static_cast *>(this); +} +#endif + #ifdef RECYCLER_WRITE_BARRIER template SmallNormalWithBarrierHeapBlockT * diff --git a/lib/Common/Memory/HeapBlock.h b/lib/Common/Memory/HeapBlock.h index 1e80fbbb57e..11e2fba0e14 100644 --- a/lib/Common/Memory/HeapBlock.h +++ b/lib/Common/Memory/HeapBlock.h @@ -73,7 +73,7 @@ enum ObjectInfoBits : unsigned short FinalizeBit = 0x80, // Indicates that the object has a finalizer PendingDisposeBit = 0x40, // Indicates that the object is pending dispose LeafBit = 0x20, // Indicates that the object is a leaf-object (objects without this bit need to be scanned) - TrackBit = 0x10, // Indicates that the object is a TrackableObject + TrackBit = 0x10, // Indicates that the object is a TrackableObject, but has also been overloaded to mean traced for RecyclerVisitedHostHeap objects ImplicitRootBit = 0x08, NewTrackBit = 0x04, // Tracked object is newly allocated and hasn't been process by concurrent GC MemoryProfilerOldObjectBit = 0x02, @@ -91,13 +91,17 @@ enum ObjectInfoBits : unsigned short WithBarrierBit = 0x0100, #endif +#ifdef RECYCLER_VISITED_HOST + RecyclerVisitedHostBit = 0x0200, +#endif + // Mask for above bits - InternalObjectInfoBitMask = 0x01FF, + InternalObjectInfoBitMask = 0x03FF, // Bits that only affect allocation behavior, not mark/sweep/etc - ClientTrackedBit = 0x0200, // This allocation is client tracked - TraceBit = 0x0400, + ClientTrackedBit = 0x0400, // This allocation is client tracked + TraceBit = 0x0800, // Additional definitions based on above @@ -134,11 +138,34 @@ enum ObjectInfoBits : unsigned short // Pending dispose objects should have LeafBit set and no others PendingDisposeObjectBits = PendingDisposeBit | LeafBit, +#ifdef RECYCLER_VISITED_HOST + // Bits for use with recycler visited host heap block. + // Recycler visited host heap block will both mark and finalize based on IRecyclerVisitedHost v-table, as specified + // by TrackBit and FinalizeBit. These objects are expected to be allocated by chakra, but implemented by + // the host, including construction of the IRecyclerVisitedHost v-table. + // + // RecyclerVisitedHostBit is implicit in the heap block type and thus isn't part of the StoredObjectInfoBitMask. + // LeafBit is also set for any object that is not precisely traced. + RecyclerVisitedHostTracedBits = RecyclerVisitedHostBit | TrackBit | NewTrackBit, + RecyclerVisitedHostFinalizableBits = RecyclerVisitedHostBit | LeafBit | FinalizeBit | NewFinalizeBit, + RecyclerVisitedHostTracedFinalizableBits = RecyclerVisitedHostTracedBits | FinalizeBit, + + // These set of bits describe the four possible types of blocktype bits for recycler visited host heap blocks. + // These are the four combinations of the above bits, AND'd with GetBlockTypeBitMask. + // In the end, these are treated the same in terms of which heap block/bucket type they end up using and + // but are defined here for ease of use. + RecyclerVisitedHostFinalizableBlockTypeBits = RecyclerVisitedHostBit | LeafBit | FinalizeBit, + RecyclerVisitedHostTracedFinalizableBlockTypeBits = RecyclerVisitedHostBit | FinalizeBit, +#endif + + GetBlockTypeBitMask = FinalizeBit | LeafBit #ifdef RECYCLER_WRITE_BARRIER - GetBlockTypeBitMask = FinalizeBit | LeafBit | WithBarrierBit, -#else - GetBlockTypeBitMask = FinalizeBit | LeafBit, + | WithBarrierBit #endif +#ifdef RECYCLER_VISITED_HOST + | RecyclerVisitedHostBit +#endif + , CollectionBitMask = LeafBit | FinalizeBit | TrackBit | NewTrackBit, // Bits relevant to collection @@ -187,32 +214,41 @@ enum FindHeapObjectFlags template class SmallNormalHeapBlockT; template class SmallLeafHeapBlockT; template class SmallFinalizableHeapBlockT; + #ifdef RECYCLER_WRITE_BARRIER template class SmallNormalWithBarrierHeapBlockT; template class SmallFinalizableWithBarrierHeapBlockT; -#define EXPLICIT_INSTANTIATE_WITH_SMALL_HEAP_BLOCK_TYPE(TemplateType) \ - template class TemplateType; \ - template class TemplateType; \ - template class TemplateType; \ +#define INSTANTIATE_SWB_BLOCKTYPES(TemplateType) \ template class TemplateType; \ template class TemplateType; \ - template class TemplateType; \ - template class TemplateType; \ - template class TemplateType; \ template class TemplateType; \ template class TemplateType; \ + +#else +#define INSTANTIATE_SWB_BLOCKTYPES(TemplateType) +#endif + +#ifdef RECYCLER_VISITED_HOST +template class SmallRecyclerVisitedHostHeapBlockT; +#define INSTANTIATE_RECYCLER_VISITED_BLOCKTYPES(TemplateType) \ + template class TemplateType; \ + template class TemplateType; \ + #else +#define INSTANTIATE_RECYCLER_VISITED_BLOCKTYPES(TemplateType) +#endif + #define EXPLICIT_INSTANTIATE_WITH_SMALL_HEAP_BLOCK_TYPE(TemplateType) \ - template class TemplateType; \ + template class TemplateType; \ template class TemplateType; \ template class TemplateType; \ - template class TemplateType; \ + template class TemplateType; \ template class TemplateType; \ template class TemplateType; \ - -#endif + INSTANTIATE_SWB_BLOCKTYPES(TemplateType) \ + INSTANTIATE_RECYCLER_VISITED_BLOCKTYPES(TemplateType) \ class RecyclerHeapObjectInfo; class HeapBlock @@ -221,31 +257,58 @@ class HeapBlock enum HeapBlockType : byte { FreeBlockType = 0, // Only used in HeapBlockMap. Actual HeapBlock structures should never have this. - SmallNormalBlockType = 1, - SmallLeafBlockType = 2, - SmallFinalizableBlockType = 3, + SmallNormalBlockType, + SmallLeafBlockType, + SmallFinalizableBlockType, #ifdef RECYCLER_WRITE_BARRIER - SmallNormalBlockWithBarrierType = 4, - SmallFinalizableBlockWithBarrierType = 5, + SmallNormalBlockWithBarrierType, + SmallFinalizableBlockWithBarrierType, #endif - MediumNormalBlockType = 6, - MediumLeafBlockType = 7, - MediumFinalizableBlockType = 8, +#ifdef RECYCLER_VISITED_HOST + SmallRecyclerVisitedHostBlockType, +#endif + MediumNormalBlockType, + MediumLeafBlockType, + MediumFinalizableBlockType, #ifdef RECYCLER_WRITE_BARRIER - MediumNormalBlockWithBarrierType = 9, - MediumFinalizableBlockWithBarrierType = 10, + MediumNormalBlockWithBarrierType, + MediumFinalizableBlockWithBarrierType, +#endif +#ifdef RECYCLER_VISITED_HOST + MediumRecyclerVisitedHostBlockType, #endif - LargeBlockType = 11, + LargeBlockType, - SmallAllocBlockTypeCount = 6, // Actual number of types for blocks containing small allocations - MediumAllocBlockTypeCount = 5, // Actual number of types for blocks containing medium allocations - SmallBlockTypeCount = 11, // Distinct block types independent of allocation size using SmallHeapBlockT +#ifdef RECYCLER_VISITED_HOST + SmallAllocBlockTypeCount = 7, // Actual number of types for blocks containing small allocations +#else + SmallAllocBlockTypeCount = 6, +#endif + +#ifdef RECYCLER_VISITED_HOST + MediumAllocBlockTypeCount = 6, // Actual number of types for blocks containing medium allocations +#else + MediumAllocBlockTypeCount = 5, +#endif - BlockTypeCount = 12, + SmallBlockTypeCount = SmallAllocBlockTypeCount + MediumAllocBlockTypeCount, // Distinct block types independent of allocation size using SmallHeapBlockT + LargeBlockTypeCount = 1, // There is only one LargeBlockType + + BlockTypeCount = SmallBlockTypeCount + LargeBlockTypeCount, }; bool IsNormalBlock() const { return this->GetHeapBlockType() == SmallNormalBlockType || this->GetHeapBlockType() == MediumNormalBlockType; } bool IsLeafBlock() const { return this->GetHeapBlockType() == SmallLeafBlockType || this->GetHeapBlockType() == MediumLeafBlockType; } - bool IsFinalizableBlock() const { return this->GetHeapBlockType() == SmallFinalizableBlockType || this->GetHeapBlockType() == MediumFinalizableBlockType; } + bool IsFinalizableBlock() const + { + return this->GetHeapBlockType() == SmallFinalizableBlockType || this->GetHeapBlockType() == MediumFinalizableBlockType +#ifdef RECYCLER_VISITED_HOST + || IsRecyclerVisitedHostBlock() +#endif + ; + } +#ifdef RECYCLER_VISITED_HOST + bool IsRecyclerVisitedHostBlock() const { return this->GetHeapBlockType() == SmallRecyclerVisitedHostBlockType || this->GetHeapBlockType() == MediumRecyclerVisitedHostBlockType; } +#endif #ifdef RECYCLER_WRITE_BARRIER bool IsAnyNormalBlock() const { return IsNormalBlock() || IsNormalWriteBarrierBlock(); } @@ -269,6 +332,12 @@ class HeapBlock template SmallFinalizableHeapBlockT * AsFinalizableBlock(); + +#ifdef RECYCLER_VISITED_HOST + template + SmallRecyclerVisitedHostHeapBlockT * AsRecyclerVisitedHostBlock(); +#endif + #ifdef RECYCLER_WRITE_BARRIER template SmallNormalWithBarrierHeapBlockT * AsNormalWriteBarrierBlock(); @@ -295,6 +364,7 @@ class HeapBlock heapBlockType(heapBlockType), needOOMRescan(false) { + static_assert(HeapBlockType::LargeBlockType == HeapBlockType::SmallBlockTypeCount, "LargeBlockType must come right after small+medium alloc block types"); Assert(GetHeapBlockType() <= HeapBlock::HeapBlockType::BlockTypeCount); } diff --git a/lib/Common/Memory/HeapBlock.inl b/lib/Common/Memory/HeapBlock.inl index cfc3ceaf6fd..d118fcd3167 100644 --- a/lib/Common/Memory/HeapBlock.inl +++ b/lib/Common/Memory/HeapBlock.inl @@ -157,6 +157,10 @@ template bool HeapBlock::UpdateAttributesOfMarkedObjects(MarkContext * markContext, void * objectAddress, size_t objectSize, unsigned char attributes, Fn fn) { +#ifdef RECYCLER_VISITED_HOST + Assert(GetHeapBlockType() != HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType && GetHeapBlockType() != HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType); +#endif + bool noOOMDuringMark = true; if (attributes & TrackBit) diff --git a/lib/Common/Memory/HeapBlockMap.cpp b/lib/Common/Memory/HeapBlockMap.cpp index 9b611814acd..2dca100b6ed 100644 --- a/lib/Common/Memory/HeapBlockMap.cpp +++ b/lib/Common/Memory/HeapBlockMap.cpp @@ -662,6 +662,10 @@ HeapBlockMap32::RescanPage(void * dirtyPage, bool* anyObjectsMarkedOnPage, Recyc case HeapBlock::HeapBlockType::SmallFinalizableBlockWithBarrierType: #endif return RescanHeapBlock(dirtyPage, blockType, chunk, id2, anyObjectsMarkedOnPage, recycler); +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType: + return RescanHeapBlock(dirtyPage, blockType, chunk, id2, anyObjectsMarkedOnPage, recycler); +#endif case HeapBlock::HeapBlockType::MediumNormalBlockType: #ifdef RECYCLER_WRITE_BARRIER case HeapBlock::HeapBlockType::MediumNormalBlockWithBarrierType: @@ -672,6 +676,10 @@ HeapBlockMap32::RescanPage(void * dirtyPage, bool* anyObjectsMarkedOnPage, Recyc case HeapBlock::HeapBlockType::MediumFinalizableBlockWithBarrierType: #endif return RescanHeapBlock(dirtyPage, blockType, chunk, id2, anyObjectsMarkedOnPage, recycler); +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType: + return RescanHeapBlock(dirtyPage, blockType, chunk, id2, anyObjectsMarkedOnPage, recycler); +#endif default: // Shouldn't be here -- leaf blocks aren't rescanned, and large blocks are handled separately Assert(false); @@ -743,6 +751,24 @@ HeapBlockMap32::GetHeapBlockForRescan(HeapBlockMap32::L2MapChunk* chunk, uint id return (MediumFinalizableHeapBlock*)chunk->map[id2]; } +#ifdef RECYCLER_VISITED_HOST +template <> +SmallRecyclerVisitedHostHeapBlock* +HeapBlockMap32::GetHeapBlockForRescan(HeapBlockMap32::L2MapChunk* chunk, uint id2) const +{ + return (SmallRecyclerVisitedHostHeapBlock*) chunk->map[id2]; +} +#endif + +#ifdef RECYCLER_VISITED_HOST +template <> +MediumRecyclerVisitedHostHeapBlock* +HeapBlockMap32::GetHeapBlockForRescan(HeapBlockMap32::L2MapChunk* chunk, uint id2) const +{ + return (MediumRecyclerVisitedHostHeapBlock*)chunk->map[id2]; +} +#endif + void HeapBlockMap32::MakeAllPagesReadOnly(Recycler* recycler) { @@ -1022,6 +1048,14 @@ HeapBlockMap32::OOMRescan(Recycler * recycler) return; } break; +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType: + if (!RescanHeapBlockOnOOM((SmallRecyclerVisitedHostHeapBlock*) heapBlock, pageAddress, blockType, chunk->blockInfo[id2].bucketIndex, chunk, recycler)) + { + return; + } + break; +#endif case HeapBlock::HeapBlockType::MediumNormalBlockType: #ifdef RECYCLER_WRITE_BARRIER @@ -1042,6 +1076,14 @@ HeapBlockMap32::OOMRescan(Recycler * recycler) return; } break; +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType: + if (!RescanHeapBlockOnOOM((MediumRecyclerVisitedHostHeapBlock*) heapBlock, pageAddress, blockType, chunk->blockInfo[id2].bucketIndex, chunk, recycler)) + { + return; + } + break; +#endif default: // Shouldn't be here -- leaf blocks aren't rescanned, and large blocks are handled separately diff --git a/lib/Common/Memory/HeapBlockMap.inl b/lib/Common/Memory/HeapBlockMap.inl index d9247fd2f72..7f9fe8695c1 100644 --- a/lib/Common/Memory/HeapBlockMap.inl +++ b/lib/Common/Memory/HeapBlockMap.inl @@ -142,13 +142,38 @@ HeapBlockMap32::Mark(void * candidate, MarkContext * markContext) #endif ((SmallFinalizableHeapBlock*)chunk->map[id2])->ProcessMarkedObject(candidate, markContext); break; +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType: + { + void * realCandidate = ((SmallFinalizableHeapBlock*)chunk->map[id2])->GetRealAddressFromInterior(candidate); + if (MarkInteriorInternal(markContext, chunk, candidate, realCandidate)) + { + break; + } + + ((SmallRecyclerVisitedHostHeapBlock*)chunk->map[id2])->ProcessMarkedObject(realCandidate, markContext); + } + break; +#endif case HeapBlock::HeapBlockType::MediumFinalizableBlockType: #ifdef RECYCLER_WRITE_BARRIER case HeapBlock::HeapBlockType::MediumFinalizableBlockWithBarrierType: #endif ((MediumFinalizableHeapBlock*)chunk->map[id2])->ProcessMarkedObject(candidate, markContext); break; +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType: + { + void * realCandidate = ((MediumFinalizableHeapBlock*)chunk->map[id2])->GetRealAddressFromInterior(candidate); + if (MarkInteriorInternal(markContext, chunk, candidate, realCandidate)) + { + break; + } + ((MediumRecyclerVisitedHostHeapBlock*)chunk->map[id2])->ProcessMarkedObject(realCandidate, markContext); + } + break; +#endif case HeapBlock::HeapBlockType::LargeBlockType: ((LargeHeapBlock*)chunk->map[id2])->Mark(candidate, markContext); break; @@ -367,7 +392,30 @@ HeapBlockMap32::MarkInterior(void * candidate, MarkContext * markContext) ((MediumFinalizableHeapBlock*)chunk->map[id2])->ProcessMarkedObject(realCandidate, markContext); } break; +#ifdef RECYCLER_VISITED_HOST + case HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType: + { + void * realCandidate = ((SmallFinalizableHeapBlock*)chunk->map[id2])->GetRealAddressFromInterior(candidate); + if (MarkInteriorInternal(markContext, chunk, candidate, realCandidate)) + { + break; + } + + ((SmallRecyclerVisitedHostHeapBlock*)chunk->map[id2])->ProcessMarkedObject(realCandidate, markContext); + } + break; + case HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType: + { + void * realCandidate = ((MediumFinalizableHeapBlock*)chunk->map[id2])->GetRealAddressFromInterior(candidate); + if (MarkInteriorInternal(markContext, chunk, candidate, realCandidate)) + { + break; + } + ((MediumRecyclerVisitedHostHeapBlock*)chunk->map[id2])->ProcessMarkedObject(realCandidate, markContext); + } + break; +#endif case HeapBlock::HeapBlockType::LargeBlockType: { void * realCandidate = ((LargeHeapBlock*)chunk->map[id2])->GetRealAddressFromInterior(candidate); diff --git a/lib/Common/Memory/HeapBucket.cpp b/lib/Common/Memory/HeapBucket.cpp index f98a5865f95..ba8bd30e23b 100644 --- a/lib/Common/Memory/HeapBucket.cpp +++ b/lib/Common/Memory/HeapBucket.cpp @@ -1588,6 +1588,9 @@ HeapBucketGroup::Initialize(HeapInfo * heapInfo, uint sizeCat) smallFinalizableWithBarrierHeapBucket.Initialize(heapInfo, sizeCat); #endif finalizableHeapBucket.Initialize(heapInfo, sizeCat); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.Initialize(heapInfo, sizeCat); +#endif } template @@ -1604,6 +1607,9 @@ HeapBucketGroup::ResetMarks(ResetMarkFlags flags) // Although we pass in premarkFreeObjects, the finalizable heap bucket ignores // this parameter and never pre-marks free objects finalizableHeapBucket.ResetMarks(flags); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.ResetMarks(flags); +#endif } template @@ -1651,6 +1657,9 @@ void HeapBucketGroup::SweepFinalizableObjects(RecyclerSweep& recyclerSweep) { finalizableHeapBucket.Sweep(recyclerSweep); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.Sweep(recyclerSweep); +#endif #ifdef RECYCLER_WRITE_BARRIER smallFinalizableWithBarrierHeapBucket.Sweep(recyclerSweep); #endif @@ -1661,6 +1670,9 @@ void HeapBucketGroup::DisposeObjects() { finalizableHeapBucket.DisposeObjects(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.DisposeObjects(); +#endif #ifdef RECYCLER_WRITE_BARRIER smallFinalizableWithBarrierHeapBucket.DisposeObjects(); #endif @@ -1671,6 +1683,9 @@ void HeapBucketGroup::TransferDisposedObjects() { finalizableHeapBucket.TransferDisposedObjects(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.TransferDisposedObjects(); +#endif #ifdef RECYCLER_WRITE_BARRIER smallFinalizableWithBarrierHeapBucket.TransferDisposedObjects(); #endif @@ -1687,6 +1702,9 @@ HeapBucketGroup::EnumerateObjects(ObjectInfoBits infoBits, voi smallFinalizableWithBarrierHeapBucket.EnumerateObjects(infoBits, CallBackFunction); #endif finalizableHeapBucket.EnumerateObjects(infoBits, CallBackFunction); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.EnumerateObjects(infoBits, CallBackFunction); +#endif } template @@ -1694,6 +1712,9 @@ void HeapBucketGroup::FinalizeAllObjects() { finalizableHeapBucket.FinalizeAllObjects(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.FinalizeAllObjects(); +#endif #ifdef RECYCLER_WRITE_BARRIER smallFinalizableWithBarrierHeapBucket.FinalizeAllObjects(); #endif @@ -1708,6 +1729,9 @@ HeapBucketGroup::Rescan(Recycler * recycler, RescanFlags flags #ifdef RECYCLER_WRITE_BARRIER smallNormalWithBarrierHeapBucket.Rescan(recycler, flags) + smallFinalizableWithBarrierHeapBucket.Rescan(recycler, flags) + +#endif +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.Rescan(recycler, flags) + #endif finalizableHeapBucket.Rescan(recycler, flags); } @@ -1724,6 +1748,9 @@ HeapBucketGroup::PrepareSweep() smallFinalizableWithBarrierHeapBucket.PrepareSweep(); #endif finalizableHeapBucket.PrepareSweep(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.PrepareSweep(); +#endif } template @@ -1752,6 +1779,9 @@ HeapBucketGroup::SweepPartialReusePages(RecyclerSweep& recycle #endif finalizableHeapBucket.SweepPartialReusePages(recyclerSweep); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.SweepPartialReusePages(recyclerSweep); +#endif } template @@ -1764,6 +1794,9 @@ HeapBucketGroup::FinishPartialCollect(RecyclerSweep * recycler smallFinalizableWithBarrierHeapBucket.FinishPartialCollect(recyclerSweep); #endif finalizableHeapBucket.FinishPartialCollect(recyclerSweep); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.FinishPartialCollect(recyclerSweep); +#endif // Leaf heap block always do a full sweep instead of partial sweep // (since touching the page doesn't affect rescan) @@ -1790,6 +1823,9 @@ HeapBucketGroup::SweepPendingObjects(RecyclerSweep& recyclerSw #endif finalizableHeapBucket.SweepPendingObjects(recyclerSweep); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.SweepPendingObjects(recyclerSweep); +#endif } template @@ -1803,6 +1839,9 @@ HeapBucketGroup::TransferPendingEmptyHeapBlocks(RecyclerSweep& recyclerSweep.TransferPendingEmptyHeapBlocks(&smallFinalizableWithBarrierHeapBucket); #endif recyclerSweep.TransferPendingEmptyHeapBlocks(&finalizableHeapBucket); +#ifdef RECYCLER_VISITED_HOST + recyclerSweep.TransferPendingEmptyHeapBlocks(&recyclerVisitedHostHeapBucket); +#endif } #endif @@ -1813,6 +1852,9 @@ HeapBucketGroup::GetNonEmptyHeapBlockCount(bool checkCount) co { return heapBucket.GetNonEmptyHeapBlockCount(checkCount) + finalizableHeapBucket.GetNonEmptyHeapBlockCount(checkCount) + +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.GetNonEmptyHeapBlockCount(checkCount) + +#endif #ifdef RECYCLER_WRITE_BARRIER smallNormalWithBarrierHeapBucket.GetNonEmptyHeapBlockCount(checkCount) + smallFinalizableWithBarrierHeapBucket.GetNonEmptyHeapBlockCount(checkCount) + @@ -1826,6 +1868,9 @@ HeapBucketGroup::GetEmptyHeapBlockCount() const { return heapBucket.GetEmptyHeapBlockCount() + finalizableHeapBucket.GetEmptyHeapBlockCount() + +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.GetEmptyHeapBlockCount() + +#endif #ifdef RECYCLER_WRITE_BARRIER smallNormalWithBarrierHeapBucket.GetEmptyHeapBlockCount() + smallFinalizableWithBarrierHeapBucket.GetEmptyHeapBlockCount() + @@ -1840,6 +1885,9 @@ size_t HeapBucketGroup::Check() { return heapBucket.Check() + finalizableHeapBucket.Check() + leafHeapBucket.Check() +#ifdef RECYCLER_VISITED_HOST + + recyclerVisitedHostHeapBucket.Check() +#endif #ifdef RECYCLER_WRITE_BARRIER + smallNormalWithBarrierHeapBucket.Check() + smallFinalizableWithBarrierHeapBucket.Check() #endif @@ -1853,6 +1901,9 @@ HeapBucketGroup::Verify() { heapBucket.Verify(); finalizableHeapBucket.Verify(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.Verify(); +#endif leafHeapBucket.Verify(); #ifdef RECYCLER_WRITE_BARRIER smallNormalWithBarrierHeapBucket.Verify(); @@ -1867,6 +1918,9 @@ HeapBucketGroup::VerifyMark() { heapBucket.VerifyMark(); finalizableHeapBucket.VerifyMark(); +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostHeapBucket.VerifyMark(); +#endif leafHeapBucket.VerifyMark(); #ifdef RECYCLER_WRITE_BARRIER smallNormalWithBarrierHeapBucket.VerifyMark(); @@ -1918,6 +1972,9 @@ HeapBucketGroup::AllocatorsAreEmpty() { return heapBucket.AllocatorsAreEmpty() && finalizableHeapBucket.AllocatorsAreEmpty() +#ifdef RECYCLER_VISITED_HOST + && recyclerVisitedHostHeapBucket.AllocatorsAreEmpty() +#endif && leafHeapBucket.AllocatorsAreEmpty() #ifdef RECYCLER_WRITE_BARRIER && smallNormalWithBarrierHeapBucket.AllocatorsAreEmpty() diff --git a/lib/Common/Memory/HeapBucket.h b/lib/Common/Memory/HeapBucket.h index a422b2013df..dc560b294fa 100644 --- a/lib/Common/Memory/HeapBucket.h +++ b/lib/Common/Memory/HeapBucket.h @@ -101,6 +101,11 @@ class HeapBucket template friend class SmallFinalizableHeapBlockT; +#ifdef RECYCLER_VISITED_HOST + template + friend class SmallRecyclerVisitedHostHeapBlockT; +#endif + friend class LargeHeapBlock; #ifdef RECYCLER_WRITE_BARRIER template @@ -164,7 +169,12 @@ class HeapBucketT : public HeapBucket protected: static bool const IsLeafBucket = TBlockType::RequiredAttributes == LeafBit; - static bool const IsFinalizableBucket = TBlockType::RequiredAttributes == FinalizeBit; + // Not all objects in the recycler visited host heap block are finalizable, but we still require finalizable semantics + static bool const IsFinalizableBucket = TBlockType::RequiredAttributes == FinalizeBit +#ifdef RECYCLER_VISITED_HOST + || ((TBlockType::RequiredAttributes & RecyclerVisitedHostBit) == (RecyclerVisitedHostBit)) +#endif + ; static bool const IsNormalBucket = TBlockType::RequiredAttributes == NoBit; #ifdef RECYCLER_WRITE_BARRIER static bool const IsWriteBarrierBucket = TBlockType::RequiredAttributes == WithBarrierBit; diff --git a/lib/Common/Memory/HeapInfo.cpp b/lib/Common/Memory/HeapInfo.cpp index 61c69f9b6bd..c468bc0b5c9 100644 --- a/lib/Common/Memory/HeapInfo.cpp +++ b/lib/Common/Memory/HeapInfo.cpp @@ -446,11 +446,17 @@ HeapInfo::HeapInfo() : newFinalizableWithBarrierHeapBlockList(nullptr), #endif newFinalizableHeapBlockList(nullptr), +#ifdef RECYCLER_VISITED_HOST + newRecyclerVisitedHostHeapBlockList(nullptr), +#endif newMediumLeafHeapBlockList(nullptr), newMediumNormalHeapBlockList(nullptr), #ifdef RECYCLER_WRITE_BARRIER newMediumNormalWithBarrierHeapBlockList(nullptr), newMediumFinalizableWithBarrierHeapBlockList(nullptr), +#endif +#ifdef RECYCLER_VISITED_HOST + newMediumRecyclerVisitedHostHeapBlockList(nullptr), #endif newMediumFinalizableHeapBlockList(nullptr), #endif @@ -498,6 +504,10 @@ HeapInfo::~HeapInfo() #if ENABLE_CONCURRENT_GC SmallFinalizableHeapBucket::FinalizeHeapBlockList(this->newFinalizableHeapBlockList); MediumFinalizableHeapBucket::FinalizeHeapBlockList(this->newMediumFinalizableHeapBlockList); +#ifdef RECYCLER_VISITED_HOST + SmallRecyclerVisitedHostHeapBucket::FinalizeHeapBlockList(this->newRecyclerVisitedHostHeapBlockList); + MediumRecyclerVisitedHostHeapBucket::FinalizeHeapBlockList(this->newMediumRecyclerVisitedHostHeapBlockList); +#endif #ifdef RECYCLER_WRITE_BARRIER SmallFinalizableWithBarrierHeapBucket::FinalizeHeapBlockList(this->newFinalizableWithBarrierHeapBlockList); MediumFinalizableWithBarrierHeapBucket::FinalizeHeapBlockList(this->newMediumFinalizableWithBarrierHeapBlockList); @@ -541,6 +551,12 @@ HeapInfo::~HeapInfo() MediumFinalizableWithBarrierHeapBucket::DeleteHeapBlockList(this->newMediumFinalizableWithBarrierHeapBlockList, recycler); #endif MediumFinalizableHeapBucket::DeleteHeapBlockList(this->newMediumFinalizableHeapBlockList, recycler); + +#ifdef RECYCLER_VISITED_HOST + SmallFinalizableHeapBucket::DeleteHeapBlockList(this->newRecyclerVisitedHostHeapBlockList, recycler); + MediumFinalizableHeapBucket::DeleteHeapBlockList(this->newMediumRecyclerVisitedHostHeapBlockList, recycler); +#endif + #endif // We do this here, instead of in the Recycler destructor, because the above stuff may @@ -781,7 +797,6 @@ HeapInfo::ResetMarks(ResetMarkFlags flags) heapBlock->MarkImplicitRoots(); }); #endif - HeapBlockList::ForEach(newMediumFinalizableHeapBlockList, [flags](MediumNormalHeapBlock * heapBlock) { heapBlock->MarkImplicitRoots(); @@ -1028,10 +1043,15 @@ HeapInfo::Sweep(RecyclerSweep& recyclerSweep, bool concurrent) // Merge the new blocks before we sweep the finalizable object in thread recyclerSweep.MergePendingNewHeapBlockList(); recyclerSweep.MergePendingNewMediumHeapBlockList(); + #ifdef RECYCLER_WRITE_BARRIER recyclerSweep.MergePendingNewHeapBlockList(); recyclerSweep.MergePendingNewMediumHeapBlockList(); #endif +#ifdef RECYCLER_VISITED_HOST + recyclerSweep.MergePendingNewHeapBlockList(); + recyclerSweep.MergePendingNewMediumHeapBlockList(); +#endif #endif SweepBuckets(recyclerSweep, concurrent); @@ -1462,6 +1482,9 @@ HeapInfo::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(voi HeapBucket::EnumerateObjects(newFinalizableWithBarrierHeapBlockList, infoBits, CallBackFunction); #endif +#ifdef RECYCLER_VISITED_HOST + HeapBucket::EnumerateObjects(newRecyclerVisitedHostHeapBlockList, infoBits, CallBackFunction); +#endif HeapBucket::EnumerateObjects(newFinalizableHeapBlockList, infoBits, CallBackFunction); HeapBucket::EnumerateObjects(newMediumLeafHeapBlockList, infoBits, CallBackFunction); @@ -1471,6 +1494,9 @@ HeapInfo::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(voi HeapBucket::EnumerateObjects(newMediumFinalizableWithBarrierHeapBlockList, infoBits, CallBackFunction); #endif +#ifdef RECYCLER_VISITED_HOST + HeapBucket::EnumerateObjects(newMediumRecyclerVisitedHostHeapBlockList, infoBits, CallBackFunction); +#endif HeapBucket::EnumerateObjects(newMediumFinalizableHeapBlockList, infoBits, CallBackFunction); #endif } @@ -1498,6 +1524,9 @@ HeapInfo::GetSmallHeapBlockCount(bool checkCount) const currentSmallHeapBlockCount += HeapBlockList::Count(this->newLeafHeapBlockList); currentSmallHeapBlockCount += HeapBlockList::Count(this->newNormalHeapBlockList); currentSmallHeapBlockCount += HeapBlockList::Count(this->newFinalizableHeapBlockList); +#ifdef RECYCLER_VISITED_HOST + currentSmallHeapBlockCount += HeapBlockList::Count(this->newRecyclerVisitedHostHeapBlockList); +#endif #ifdef RECYCLER_WRITE_BARRIER currentSmallHeapBlockCount += HeapBlockList::Count(this->newNormalWithBarrierHeapBlockList); currentSmallHeapBlockCount += HeapBlockList::Count(this->newFinalizableWithBarrierHeapBlockList); @@ -1505,6 +1534,9 @@ HeapInfo::GetSmallHeapBlockCount(bool checkCount) const currentSmallHeapBlockCount += HeapBlockList::Count(this->newMediumLeafHeapBlockList); currentSmallHeapBlockCount += HeapBlockList::Count(this->newMediumNormalHeapBlockList); +#ifdef RECYCLER_VISITED_HOST + currentSmallHeapBlockCount += HeapBlockList::Count(this->newMediumRecyclerVisitedHostHeapBlockList); +#endif currentSmallHeapBlockCount += HeapBlockList::Count(this->newMediumFinalizableHeapBlockList); #ifdef RECYCLER_WRITE_BARRIER currentSmallHeapBlockCount += HeapBlockList::Count(this->newMediumNormalWithBarrierHeapBlockList); @@ -1525,6 +1557,10 @@ HeapInfo::GetSmallHeapBlockCount(bool checkCount) const this->heapBlockCount[HeapBlock::HeapBlockType::SmallNormalBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::SmallLeafBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::SmallFinalizableBlockType] +#ifdef RECYCLER_VISITED_HOST + + this->heapBlockCount[HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType] + + this->heapBlockCount[HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType] +#endif + this->heapBlockCount[HeapBlock::HeapBlockType::MediumNormalBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::MediumLeafBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::MediumFinalizableBlockType]; @@ -1598,6 +1634,10 @@ HeapInfo::Check() currentSmallHeapBlockCount += Check(true, false, this->newFinalizableWithBarrierHeapBlockList); #endif currentSmallHeapBlockCount += Check(true, false, this->newFinalizableHeapBlockList); +#ifdef RECYCLER_VISITED_HOST + currentSmallHeapBlockCount += Check(true, false, this->newRecyclerVisitedHostHeapBlockList); + currentSmallHeapBlockCount += Check(true, false, this->newMediumRecyclerVisitedHostHeapBlockList); +#endif #endif #if ENABLE_CONCURRENT_GC @@ -1614,6 +1654,10 @@ HeapInfo::Check() this->heapBlockCount[HeapBlock::HeapBlockType::SmallNormalBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::SmallLeafBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::SmallFinalizableBlockType] +#ifdef RECYCLER_VISITED_HOST + + this->heapBlockCount[HeapBlock::HeapBlockType::SmallRecyclerVisitedHostBlockType] + + this->heapBlockCount[HeapBlock::HeapBlockType::MediumRecyclerVisitedHostBlockType] +#endif + this->heapBlockCount[HeapBlock::HeapBlockType::MediumNormalBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::MediumLeafBlockType] + this->heapBlockCount[HeapBlock::HeapBlockType::MediumFinalizableBlockType]; @@ -1650,6 +1694,10 @@ HeapInfo::Check(bool expectFull, bool expectPending, TBlockType * list, TBlockTy template size_t HeapInfo::Check(bool expectFull, bool expectPending, SmallNormalHeapBlock * list, SmallNormalHeapBlock * tail); template size_t HeapInfo::Check(bool expectFull, bool expectPending, SmallLeafHeapBlock * list, SmallLeafHeapBlock * tail); template size_t HeapInfo::Check(bool expectFull, bool expectPending, SmallFinalizableHeapBlock * list, SmallFinalizableHeapBlock * tail); +#ifdef RECYCLER_VISITED_HOST +template size_t HeapInfo::Check(bool expectFull, bool expectPending, SmallRecyclerVisitedHostHeapBlock * list, SmallRecyclerVisitedHostHeapBlock * tail); +template size_t HeapInfo::Check(bool expectFull, bool expectPending, MediumRecyclerVisitedHostHeapBlock * list, MediumRecyclerVisitedHostHeapBlock * tail); +#endif template size_t HeapInfo::Check(bool expectFull, bool expectPending, LargeHeapBlock * list, LargeHeapBlock * tail); #ifdef RECYCLER_WRITE_BARRIER template size_t HeapInfo::Check(bool expectFull, bool expectPending, SmallNormalWithBarrierHeapBlock * list, SmallNormalWithBarrierHeapBlock * tail); @@ -1714,6 +1762,12 @@ HeapInfo::Verify() { heapBlock->Verify(); }); +#endif +#ifdef RECYCLER_VISITED_HOST + HeapBlockList::ForEach(newRecyclerVisitedHostHeapBlockList, [](SmallFinalizableHeapBlock * heapBlock) + { + heapBlock->Verify(); + }); #endif HeapBlockList::ForEach(newFinalizableHeapBlockList, [](SmallFinalizableHeapBlock * heapBlock) { @@ -1739,6 +1793,12 @@ HeapInfo::Verify() { heapBlock->Verify(); }); +#endif +#ifdef RECYCLER_VISITED_HOST + HeapBlockList::ForEach(newMediumRecyclerVisitedHostHeapBlockList, [](MediumFinalizableHeapBlock * heapBlock) + { + heapBlock->Verify(); + }); #endif HeapBlockList::ForEach(newMediumFinalizableHeapBlockList, [](MediumFinalizableHeapBlock * heapBlock) { @@ -1784,6 +1844,12 @@ HeapInfo::VerifyMark() { heapBlock->VerifyMark(); }); +#endif +#ifdef RECYCLER_VISITED_HOST + HeapBlockList::ForEach(newRecyclerVisitedHostHeapBlockList, [](SmallFinalizableHeapBlock * heapBlock) + { + heapBlock->VerifyMark(); + }); #endif HeapBlockList::ForEach(newFinalizableHeapBlockList, [](SmallFinalizableHeapBlock * heapBlock) { @@ -1809,6 +1875,12 @@ HeapInfo::VerifyMark() { heapBlock->VerifyMark(); }); +#endif +#ifdef RECYCLER_VISITED_HOST + HeapBlockList::ForEach(newMediumRecyclerVisitedHostHeapBlockList, [](MediumFinalizableHeapBlock * heapBlock) + { + heapBlock->VerifyMark(); + }); #endif HeapBlockList::ForEach(newMediumFinalizableHeapBlockList, [](MediumFinalizableHeapBlock * heapBlock) { diff --git a/lib/Common/Memory/HeapInfo.h b/lib/Common/Memory/HeapInfo.h index 89c83a57e6c..31ce8ce18e1 100644 --- a/lib/Common/Memory/HeapInfo.h +++ b/lib/Common/Memory/HeapInfo.h @@ -214,6 +214,13 @@ class HeapInfo // so new block can't go into heapBlockList return this->newFinalizableHeapBlockList; } +#ifdef RECYCLER_VISITED_HOST + template <> + SmallRecyclerVisitedHostHeapBlock *& GetNewHeapBlockList(HeapBucketT * HeapBucket) + { + return this->newRecyclerVisitedHostHeapBlockList; + } +#endif #ifdef RECYCLER_WRITE_BARRIER template <> @@ -248,6 +255,14 @@ class HeapInfo return this->newMediumFinalizableHeapBlockList; } +#ifdef RECYCLER_VISITED_HOST + template <> + MediumRecyclerVisitedHostHeapBlock *& GetNewHeapBlockList(HeapBucketT * HeapBucket) + { + return this->newMediumRecyclerVisitedHostHeapBlockList; + } +#endif + #ifdef RECYCLER_WRITE_BARRIER template <> MediumNormalWithBarrierHeapBlock *& GetNewHeapBlockList(HeapBucketT * heapBucket) @@ -401,6 +416,9 @@ class HeapInfo SmallLeafHeapBlock * newLeafHeapBlockList; SmallNormalHeapBlock * newNormalHeapBlockList; SmallFinalizableHeapBlock * newFinalizableHeapBlockList; +#ifdef RECYCLER_VISITED_HOST + SmallRecyclerVisitedHostHeapBlock * newRecyclerVisitedHostHeapBlockList; +#endif #ifdef RECYCLER_WRITE_BARRIER SmallNormalWithBarrierHeapBlock * newNormalWithBarrierHeapBlockList; @@ -412,6 +430,9 @@ class HeapInfo MediumLeafHeapBlock * newMediumLeafHeapBlockList; MediumNormalHeapBlock * newMediumNormalHeapBlockList; MediumFinalizableHeapBlock * newMediumFinalizableHeapBlockList; +#ifdef RECYCLER_VISITED_HOST + MediumRecyclerVisitedHostHeapBlock* newMediumRecyclerVisitedHostHeapBlockList; +#endif #ifdef RECYCLER_WRITE_BARRIER MediumNormalWithBarrierHeapBlock * newMediumNormalWithBarrierHeapBlockList; @@ -471,6 +492,10 @@ class HeapInfo friend class SmallLeafHeapBlockT; template friend class SmallFinalizableHeapBlockT; +#ifdef RECYCLER_VISITED_HOST + template + friend class SmallRecyclerVisitedHostHeapBlockT; +#endif friend class LargeHeapBlock; friend class RecyclerSweep; }; diff --git a/lib/Common/Memory/LargeHeapBlock.cpp b/lib/Common/Memory/LargeHeapBlock.cpp index 73e777b697f..c6792de7eb0 100644 --- a/lib/Common/Memory/LargeHeapBlock.cpp +++ b/lib/Common/Memory/LargeHeapBlock.cpp @@ -504,6 +504,13 @@ LargeHeapBlock::AllocFreeListEntry(DECLSPEC_GUARD_OVERFLOW size_t size, ObjectIn Assert(entry->headerIndex < this->objectCount); Assert(this->HeaderList()[entry->headerIndex] == nullptr); +#ifdef RECYCLER_VISITED_HOST + if (attributes & RecyclerVisitedHostBit) + { + ReportFatalException(NULL, E_FAIL, Fatal_RecyclerVisitedHost_LargeHeapBlock, 1); + } +#endif + uint headerIndex = entry->headerIndex; size_t originalSize = entry->objectSize; @@ -571,6 +578,12 @@ LargeHeapBlock::Alloc(DECLSPEC_GUARD_OVERFLOW size_t size, ObjectInfoBits attrib Assert(HeapInfo::IsAlignedSize(size) || InPageHeapMode()); Assert((attributes & InternalObjectInfoBitMask) == attributes); AssertMsg((attributes & TrackBit) == 0, "Large tracked object collection not implemented"); +#ifdef RECYCLER_VISITED_HOST + if (attributes & RecyclerVisitedHostBit) + { + ReportFatalException(NULL, E_FAIL, Fatal_RecyclerVisitedHost_LargeHeapBlock, 2); + } +#endif LargeObjectHeader * header = (LargeObjectHeader *)allocAddressEnd; #if ENABLE_PARTIAL_GC && ENABLE_CONCURRENT_GC diff --git a/lib/Common/Memory/MarkContext.cpp b/lib/Common/Memory/MarkContext.cpp index 274e084c39d..0e21c95ea3f 100644 --- a/lib/Common/Memory/MarkContext.cpp +++ b/lib/Common/Memory/MarkContext.cpp @@ -14,6 +14,9 @@ MarkContext::MarkContext(Recycler * recycler, PagePool * pagePool) : recycler(recycler), pagePool(pagePool), markStack(pagePool), +#ifdef RECYCLER_VISITED_HOST + preciseStack(pagePool), +#endif trackStack(pagePool) { } @@ -39,18 +42,27 @@ void MarkContext::OnObjectMarked(void* object, void* parent) void MarkContext::Init(uint reservedPageCount) { markStack.Init(reservedPageCount); +#ifdef RECYCLER_VISITED_HOST + preciseStack.Init(); +#endif trackStack.Init(); } void MarkContext::Clear() { markStack.Clear(); +#ifdef RECYCLER_VISITED_HOST + preciseStack.Clear(); +#endif trackStack.Clear(); } void MarkContext::Abort() { markStack.Abort(); +#ifdef RECYCLER_VISITED_HOST + preciseStack.Abort(); +#endif trackStack.Abort(); pagePool->ReleaseFreePages(); @@ -60,6 +72,9 @@ void MarkContext::Abort() void MarkContext::Release() { markStack.Release(); +#ifdef RECYCLER_VISITED_HOST + preciseStack.Release(); +#endif trackStack.Release(); pagePool->ReleaseFreePages(); @@ -68,17 +83,33 @@ void MarkContext::Release() uint MarkContext::Split(uint targetCount, __in_ecount(targetCount) MarkContext ** targetContexts) { - Assert(targetCount > 0 && targetCount <= PageStack::MaxSplitTargets); + Assert(targetCount > 0 && targetCount <= PageStack::MaxSplitTargets && targetCount <= PageStack::MaxSplitTargets); __analysis_assume(targetCount <= PageStack::MaxSplitTargets); + __analysis_assume(targetCount <= PageStack::MaxSplitTargets); - PageStack * targetStacks[PageStack::MaxSplitTargets]; + PageStack * targetMarkStacks[PageStack::MaxSplitTargets]; +#ifdef RECYCLER_VISITED_HOST + PageStack * targetPreciseStacks[PageStack::MaxSplitTargets]; +#endif for (uint i = 0; i < targetCount; i++) { - targetStacks[i] = &targetContexts[i]->markStack; + targetMarkStacks[i] = &targetContexts[i]->markStack; +#ifdef RECYCLER_VISITED_HOST + targetPreciseStacks[i] = &targetContexts[i]->preciseStack; +#endif } - return this->markStack.Split(targetCount, targetStacks); + // Return the max count of the two splits - since the stacks have more or less unrelated sizes, they + // could yield different number of splits, but the caller wants to know the max parallelism it + // should use on the results of the split. + const uint markStackSplitCount = this->markStack.Split(targetCount, targetMarkStacks); +#ifdef RECYCLER_VISITED_HOST + const uint preciseStackSplitCount = this->preciseStack.Split(targetCount, targetPreciseStacks); + return max(markStackSplitCount, preciseStackSplitCount); +#else + return markStackSplitCount; +#endif } diff --git a/lib/Common/Memory/MarkContext.h b/lib/Common/Memory/MarkContext.h index 9ecb055c6e1..3cb8515fa40 100644 --- a/lib/Common/Memory/MarkContext.h +++ b/lib/Common/Memory/MarkContext.h @@ -2,6 +2,8 @@ // Copyright (C) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. //------------------------------------------------------------------------------------------------------- +interface IRecyclerVisitedObject; + namespace Memory { class Recycler; @@ -29,6 +31,9 @@ class MarkContext Recycler * GetRecycler() { return this->recycler; } bool AddMarkedObject(void * obj, size_t byteCount); +#ifdef RECYCLER_VISITED_HOST + bool AddPreciselyTracedObject(IRecyclerVisitedObject *obj); +#endif #if ENABLE_CONCURRENT_GC bool AddTrackedObject(FinalizableObject * obj); #endif @@ -53,8 +58,17 @@ class MarkContext void Release(); bool HasPendingMarkObjects() const { return !markStack.IsEmpty(); } +#ifdef RECYCLER_VISITED_HOST + bool HasPendingPreciselyTracedObjects() const { return !preciseStack.IsEmpty(); } +#endif bool HasPendingTrackObjects() const { return !trackStack.IsEmpty(); } - bool HasPendingObjects() const { return HasPendingMarkObjects() || HasPendingTrackObjects(); } + bool HasPendingObjects() const { + return HasPendingMarkObjects() +#ifdef RECYCLER_VISITED_HOST + || HasPendingPreciselyTracedObjects() +#endif + || HasPendingTrackObjects(); + } PageAllocator * GetPageAllocator() { return this->pagePool->GetPageAllocator(); } @@ -88,7 +102,13 @@ class MarkContext #ifdef ENABLE_DEBUG_CONFIG_OPTIONS - void SetMaxPageCount(size_t maxPageCount) { markStack.SetMaxPageCount(maxPageCount); trackStack.SetMaxPageCount(maxPageCount); } + void SetMaxPageCount(size_t maxPageCount) { + markStack.SetMaxPageCount(maxPageCount); +#ifdef RECYCLER_VISITED_HOST + preciseStack.SetMaxPageCount(maxPageCount); +#endif + trackStack.SetMaxPageCount(maxPageCount); + } #endif #ifdef RECYCLER_MARK_TRACK @@ -102,6 +122,9 @@ class MarkContext Recycler * recycler; PagePool * pagePool; PageStack markStack; +#ifdef RECYCLER_VISITED_HOST + PageStack preciseStack; +#endif PageStack trackStack; #ifdef RECYCLER_MARK_TRACK diff --git a/lib/Common/Memory/MarkContext.inl b/lib/Common/Memory/MarkContext.inl index 081bde3a7fb..0d0fb7c2a52 100644 --- a/lib/Common/Memory/MarkContext.inl +++ b/lib/Common/Memory/MarkContext.inl @@ -34,6 +34,15 @@ bool MarkContext::AddMarkedObject(void * objectAddress, size_t objectSize) return markStack.Push(markCandidate); } +#ifdef RECYCLER_VISITED_HOST +inline bool MarkContext::AddPreciselyTracedObject(IRecyclerVisitedObject* obj) +{ + FAULTINJECT_MEMORY_MARK_NOTHROW(_u("AddPreciselyTracedObject"), 0); + + return preciseStack.Push(obj); +} +#endif + #if ENABLE_CONCURRENT_GC inline bool MarkContext::AddTrackedObject(FinalizableObject * obj) @@ -134,12 +143,6 @@ void MarkContext::Mark(void * candidate, void * parentReference) if (interior) { -#if ENABLE_CONCURRENT_GC - Assert(recycler->enableScanInteriorPointers - || (!recycler->IsConcurrentState() && recycler->collectionState != CollectionStateParallelMark)); -#else - Assert(recycler->enableScanInteriorPointers || recycler->collectionState != CollectionStateParallelMark); -#endif recycler->heapBlockMap.MarkInterior(candidate, this); return; } @@ -194,41 +197,64 @@ void MarkContext::ProcessMark() } #endif +#ifdef RECYCLER_VISITED_HOST + // Flip between processing the generic mark stack (conservatively traced with ScanMemory) and + // the precise stack (precisely traced via IRecyclerVisitedObject::Trace). Each of those + // operations on an object has the potential to add new marked objects to either or both + // stacks so we must loop until they are both empty. + while (!markStack.IsEmpty() || !preciseStack.IsEmpty()) +#endif + { #if defined(_M_IX86) || defined(_M_X64) - MarkCandidate current, next; + MarkCandidate current, next; - while (markStack.Pop(¤t)) - { - // Process entries and prefetch as we go. - while (markStack.Pop(&next)) + while (markStack.Pop(¤t)) { - // Prefetch the next entry so it's ready when we need it. - _mm_prefetch((char *)next.obj, _MM_HINT_T0); + // Process entries and prefetch as we go. + while (markStack.Pop(&next)) + { + // Prefetch the next entry so it's ready when we need it. + _mm_prefetch((char *)next.obj, _MM_HINT_T0); - // Process the previously retrieved entry. - ScanObject(current.obj, current.byteCount); + // Process the previously retrieved entry. + ScanObject(current.obj, current.byteCount); - _mm_prefetch((char *)*(next.obj), _MM_HINT_T0); + _mm_prefetch((char *)*(next.obj), _MM_HINT_T0); - current = next; - } + current = next; + } - // The stack is empty, but we still have a previously retrieved entry; process it now. - ScanObject(current.obj, current.byteCount); + // The stack is empty, but we still have a previously retrieved entry; process it now. + ScanObject(current.obj, current.byteCount); - // Processing that entry may have generated more entries in the mark stack, so continue the loop. - } + // Processing that entry may have generated more entries in the mark stack, so continue the loop. + } #else - // _mm_prefetch intrinsic is specific to Intel platforms. - // CONSIDER: There does seem to be a compiler intrinsic for prefetch on ARM, - // however, the information on this is scarce, so for now just don't do prefetch on ARM. - MarkCandidate current; + // _mm_prefetch intrinsic is specific to Intel platforms. + // CONSIDER: There does seem to be a compiler intrinsic for prefetch on ARM, + // however, the information on this is scarce, so for now just don't do prefetch on ARM. + MarkCandidate current; - while (markStack.Pop(¤t)) - { - ScanObject(current.obj, current.byteCount); - } + while (markStack.Pop(¤t)) + { + ScanObject(current.obj, current.byteCount); + } #endif - Assert(markStack.IsEmpty()); + Assert(markStack.IsEmpty()); + +#ifdef RECYCLER_VISITED_HOST + if (!preciseStack.IsEmpty()) + { + MarkContextWrapper markContextWrapper(this); + IRecyclerVisitedObject* tracedObject; + while (preciseStack.Pop(&tracedObject)) + { + tracedObject->Trace(&markContextWrapper); + } + } + + Assert(preciseStack.IsEmpty()); +#endif + } } diff --git a/lib/Common/Memory/MarkContextWrapper.h b/lib/Common/Memory/MarkContextWrapper.h new file mode 100644 index 00000000000..7d8efb69fdb --- /dev/null +++ b/lib/Common/Memory/MarkContextWrapper.h @@ -0,0 +1,30 @@ +//------------------------------------------------------------------------------------------------------- +// Copyright (C) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. +//------------------------------------------------------------------------------------------------------- + +#include "Core/RecyclerHeapMarkingContext.h" + +// Class used to wrap a MarkContext so that calls to MarkObjects during IRecyclerVisitedObject::Trace +// can mark with the correct contextual template parameters +template +class MarkContextWrapper : public IRecyclerHeapMarkingContext +{ +public: + MarkContextWrapper(MarkContext* context) : markContext(context) {} + + void MarkObjects(void** objects, size_t count, void* parent) override + { + for (size_t i = 0; i < count; i++) + { + // We pass true for interior, since we expect a majority of the pointers being marked by + // external objects to themselves be external (and thus candidates for interior pointers). + markContext->Mark(objects[i], parent); + } + } + +private: + // Should only be created on the stack + void* operator new(size_t); + MarkContext* markContext; +}; diff --git a/lib/Common/Memory/Recycler.cpp b/lib/Common/Memory/Recycler.cpp index dc07e8a1922..ad5f0cb5438 100644 --- a/lib/Common/Memory/Recycler.cpp +++ b/lib/Common/Memory/Recycler.cpp @@ -8723,3 +8723,9 @@ RecyclerHeapObjectInfo::GetSize() const } template char* Recycler::AllocWithAttributesInlined<(Memory::ObjectInfoBits)32, false>(size_t); +#ifdef RECYCLER_VISITED_HOST +template char* Recycler::AllocZeroWithAttributesInlined(size_t); +template char* Recycler::AllocZeroWithAttributesInlined(size_t); +template char* Recycler::AllocZeroWithAttributesInlined(size_t); +template char* Recycler::AllocZeroWithAttributesInlined(size_t); +#endif \ No newline at end of file diff --git a/lib/Common/Memory/Recycler.h b/lib/Common/Memory/Recycler.h index 9a0a5fbaf13..cc2b3be7643 100644 --- a/lib/Common/Memory/Recycler.h +++ b/lib/Common/Memory/Recycler.h @@ -182,7 +182,12 @@ struct InfoBitsWrapper{}; #define RecyclerNewTrackedLeaf(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedLeafInlined, T, __VA_ARGS__))) #define RecyclerNewTrackedLeafPlusZ(recycler,size,T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocZeroTrackedLeafInlined, size, T, __VA_ARGS__))) - +#ifdef RECYCLER_VISITED_HOST +#define RecyclerAllocVisitedHostTracedAndFinalizedZero(recycler,size) recycler->AllocVisitedHost(size) +#define RecyclerAllocVisitedHostFinalizedZero(recycler,size) recycler->AllocVisitedHost(size) +#define RecyclerAllocVisitedHostTracedZero(recycler,size) recycler->AllocVisitedHost(size) +#define RecyclerAllocLeafZero(recycler,size) recycler->AllocVisitedHost(size) +#endif #ifdef TRACE_OBJECT_LIFETIME #define RecyclerNewLeafTrace(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocLeafTrace, T, __VA_ARGS__) @@ -1290,6 +1295,11 @@ class Recycler { return AllocWithAttributes(size); } + template + char * AllocVisitedHost(DECLSPEC_GUARD_OVERFLOW size_t size) + { + return AllocZeroWithAttributes(size); + } template RecyclerWeakReference* CreateWeakReferenceHandle(T* pStrongReference); @@ -1565,6 +1575,9 @@ class Recycler template void ScanMemory(void ** obj, size_t byteCount) { if (byteCount != 0) { ScanMemoryInline(obj, byteCount); } } bool AddMark(void * candidate, size_t byteCount); +#ifdef RECYCLER_VISITED_HOST + bool AddPreciselyTracedMark(IRecyclerVisitedObject * candidate); +#endif // Sweep #if ENABLE_PARTIAL_GC @@ -1717,6 +1730,9 @@ class Recycler template friend class SmallNormalHeapBlockT; template friend class SmallLeafHeapBlockT; template friend class SmallFinalizableHeapBlockT; +#ifdef RECYCLER_VISITED_HOST + template friend class SmallRecyclerVisitedHostHeapBlockT; +#endif friend class LargeHeapBlock; friend class HeapInfo; friend class LargeHeapBucket; diff --git a/lib/Common/Memory/Recycler.inl b/lib/Common/Memory/Recycler.inl index 637d05be5e9..5ee90e4990d 100644 --- a/lib/Common/Memory/Recycler.inl +++ b/lib/Common/Memory/Recycler.inl @@ -46,8 +46,12 @@ template inline char * Recycler::AllocWithAttributesInlined(DECLSPEC_GUARD_OVERFLOW size_t size) { - // All tracked objects are client tracked objects + // All tracked objects are client tracked or recycler host visited objects +#ifndef RECYCLER_VISITED_HOST CompileAssert((attributes & TrackBit) == 0 || (attributes & ClientTrackedBit) != 0); +#else + CompileAssert((attributes & TrackBit) == 0 || (attributes & ClientTrackedBit) != 0 || (attributes & RecyclerVisitedHostBit) != 0); +#endif Assert(this->enableScanImplicitRoots || (attributes & ImplicitRootBit) == 0); AssertMsg(this->disableThreadAccessCheck || this->mainThreadId == GetCurrentThreadContextId(), "Allocating from the recycler can only be done on the main thread"); @@ -517,6 +521,15 @@ Recycler::AddMark(void * candidate, size_t byteCount) throw() return markContext.AddMarkedObject(candidate, byteCount); } +#ifdef RECYCLER_VISITED_HOST +inline bool +Recycler::AddPreciselyTracedMark(IRecyclerVisitedObject * candidate) throw() +{ + // This API cannot be used for parallel marking as we don't have enough information to determine which MarkingContext to use. + Assert((this->collectionState & Collection_Parallel) == 0); + return markContext.AddPreciselyTracedObject(candidate); +} +#endif template void diff --git a/lib/Common/Memory/RecyclerSweep.cpp b/lib/Common/Memory/RecyclerSweep.cpp index ca77e4242cb..66a06644539 100644 --- a/lib/Common/Memory/RecyclerSweep.cpp +++ b/lib/Common/Memory/RecyclerSweep.cpp @@ -72,6 +72,9 @@ RecyclerSweep::BeginSweep(Recycler * recycler) finalizableWithBarrierData.pendingMergeNewHeapBlockList = recycler->autoHeap.newFinalizableWithBarrierHeapBlockList; #endif finalizableData.pendingMergeNewHeapBlockList = recycler->autoHeap.newFinalizableHeapBlockList; +#ifdef RECYCLER_VISITED_HOST + recyclerVisitedHostData.pendingMergeNewHeapBlockList = recycler->autoHeap.newRecyclerVisitedHostHeapBlockList; +#endif mediumLeafData.pendingMergeNewHeapBlockList = recycler->autoHeap.newMediumLeafHeapBlockList; mediumNormalData.pendingMergeNewHeapBlockList = recycler->autoHeap.newMediumNormalHeapBlockList; @@ -80,11 +83,17 @@ RecyclerSweep::BeginSweep(Recycler * recycler) mediumFinalizableWithBarrierData.pendingMergeNewHeapBlockList = recycler->autoHeap.newMediumFinalizableWithBarrierHeapBlockList; #endif mediumFinalizableData.pendingMergeNewHeapBlockList = recycler->autoHeap.newMediumFinalizableHeapBlockList; +#ifdef RECYCLER_VISITED_HOST + mediumRecyclerVisitedHostData.pendingMergeNewHeapBlockList = recycler->autoHeap.newMediumRecyclerVisitedHostHeapBlockList; +#endif recycler->autoHeap.newLeafHeapBlockList = nullptr; recycler->autoHeap.newNormalHeapBlockList = nullptr; recycler->autoHeap.newFinalizableHeapBlockList = nullptr; +#ifdef RECYCLER_VISITED_HOST + recycler->autoHeap.newRecyclerVisitedHostHeapBlockList = nullptr; +#endif #ifdef RECYCLER_WRITE_BARRIER recycler->autoHeap.newNormalWithBarrierHeapBlockList = nullptr; recycler->autoHeap.newFinalizableWithBarrierHeapBlockList = nullptr; @@ -93,6 +102,9 @@ RecyclerSweep::BeginSweep(Recycler * recycler) recycler->autoHeap.newMediumLeafHeapBlockList = nullptr; recycler->autoHeap.newMediumNormalHeapBlockList = nullptr; recycler->autoHeap.newMediumFinalizableHeapBlockList = nullptr; +#ifdef RECYCLER_VISITED_HOST + recycler->autoHeap.newMediumRecyclerVisitedHostHeapBlockList = nullptr; +#endif #ifdef RECYCLER_WRITE_BARRIER recycler->autoHeap.newMediumNormalWithBarrierHeapBlockList = nullptr; recycler->autoHeap.newMediumFinalizableWithBarrierHeapBlockList = nullptr; @@ -401,6 +413,9 @@ RecyclerSweep::MergePendingNewHeapBlockList() template void RecyclerSweep::MergePendingNewHeapBlockList(); template void RecyclerSweep::MergePendingNewHeapBlockList(); template void RecyclerSweep::MergePendingNewHeapBlockList(); +#ifdef RECYCLER_VISITED_HOST +template void RecyclerSweep::MergePendingNewHeapBlockList(); +#endif #ifdef RECYCLER_WRITE_BARRIER template void RecyclerSweep::MergePendingNewHeapBlockList(); template void RecyclerSweep::MergePendingNewHeapBlockList(); @@ -424,6 +439,9 @@ RecyclerSweep::MergePendingNewMediumHeapBlockList() template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); +#ifdef RECYCLER_VISITED_HOST +template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); +#endif #ifdef RECYCLER_WRITE_BARRIER template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); template void RecyclerSweep::MergePendingNewMediumHeapBlockList(); @@ -492,6 +510,10 @@ RecyclerSweep::SetPendingMergeNewHeapBlockCount() return HeapBlockList::Count(leafData.pendingMergeNewHeapBlockList) + HeapBlockList::Count(normalData.pendingMergeNewHeapBlockList) + HeapBlockList::Count(finalizableData.pendingMergeNewHeapBlockList) +#ifdef RECYCLER_VISITED_HOST + + HeapBlockList::Count(recyclerVisitedHostData.pendingMergeNewHeapBlockList) + + HeapBlockList::Count(mediumRecyclerVisitedHostData.pendingMergeNewHeapBlockList) +#endif #ifdef RECYCLER_WRITE_BARRIER + HeapBlockList::Count(withBarrierData.pendingMergeNewHeapBlockList) + HeapBlockList::Count(finalizableWithBarrierData.pendingMergeNewHeapBlockList) diff --git a/lib/Common/Memory/RecyclerSweep.h b/lib/Common/Memory/RecyclerSweep.h index 1cae6ef178e..56f8ae9555b 100644 --- a/lib/Common/Memory/RecyclerSweep.h +++ b/lib/Common/Memory/RecyclerSweep.h @@ -122,6 +122,9 @@ class RecyclerSweep template <> Data& GetData() { return leafData; } template <> Data& GetData() { return normalData; } template <> Data& GetData() { return finalizableData; } +#ifdef RECYCLER_VISITED_HOST + template <> Data& GetData() { return recyclerVisitedHostData; } +#endif #ifdef RECYCLER_WRITE_BARRIER template <> Data& GetData() { return withBarrierData; } template <> Data& GetData() { return finalizableWithBarrierData; } @@ -130,6 +133,9 @@ class RecyclerSweep template <> Data& GetData() { return mediumLeafData; } template <> Data& GetData() { return mediumNormalData; } template <> Data& GetData() { return mediumFinalizableData; } +#ifdef RECYCLER_VISITED_HOST + template <> Data& GetData() { return mediumRecyclerVisitedHostData; } +#endif #ifdef RECYCLER_WRITE_BARRIER template <> Data& GetData() { return mediumWithBarrierData; } template <> Data& GetData() { return mediumFinalizableWithBarrierData; } @@ -142,6 +148,9 @@ class RecyclerSweep Data leafData; Data normalData; Data finalizableData; +#ifdef RECYCLER_VISITED_HOST + Data recyclerVisitedHostData; +#endif #ifdef RECYCLER_WRITE_BARRIER Data withBarrierData; Data finalizableWithBarrierData; @@ -149,6 +158,9 @@ class RecyclerSweep Data mediumLeafData; Data mediumNormalData; Data mediumFinalizableData; +#ifdef RECYCLER_VISITED_HOST + Data mediumRecyclerVisitedHostData; +#endif #ifdef RECYCLER_WRITE_BARRIER Data mediumWithBarrierData; Data mediumFinalizableWithBarrierData; diff --git a/lib/Common/Memory/SmallBlockDeclarations.inl b/lib/Common/Memory/SmallBlockDeclarations.inl index b3bf397615d..2168248ebf9 100644 --- a/lib/Common/Memory/SmallBlockDeclarations.inl +++ b/lib/Common/Memory/SmallBlockDeclarations.inl @@ -18,6 +18,9 @@ template SweepState SmallHeapBlockT::Sweep(RecyclerSweep& template SmallNormalHeapBlockT* HeapBlock::AsNormalBlock(); template SmallLeafHeapBlockT* HeapBlock::AsLeafBlock(); template SmallFinalizableHeapBlockT* HeapBlock::AsFinalizableBlock(); +#ifdef RECYCLER_VISITED_HOST +template SmallRecyclerVisitedHostHeapBlockT* HeapBlock::AsRecyclerVisitedHostBlock(); +#endif #ifdef RECYCLER_WRITE_BARRIER template SmallNormalWithBarrierHeapBlockT* HeapBlock::AsNormalWriteBarrierBlock(); template SmallFinalizableWithBarrierHeapBlockT* HeapBlock::AsFinalizableWriteBarrierBlock(); @@ -26,6 +29,9 @@ template SmallFinalizableWithBarrierHeapBlockT* HeapBlock: template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); +#ifdef RECYCLER_VISITED_HOST +template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); +#endif #ifdef RECYCLER_WRITE_BARRIER template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); template bool SmallHeapBlockT::FindHeapObjectImpl>(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); @@ -35,6 +41,9 @@ template bool SmallHeapBlockT::FindHeapObjectImpl::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); template bool SmallHeapBlockT::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); template bool SmallHeapBlockT::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); +#ifdef RECYCLER_VISITED_HOST +template bool SmallHeapBlockT::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); +#endif #ifdef RECYCLER_WRITE_BARRIER template bool SmallHeapBlockT::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); template bool SmallHeapBlockT::GetFreeObjectListOnAllocatorImpl>(FreeObject ** freeObjectList); diff --git a/lib/Common/Memory/SmallFinalizableHeapBlock.cpp b/lib/Common/Memory/SmallFinalizableHeapBlock.cpp index 6806ba7022a..206cfb98e53 100644 --- a/lib/Common/Memory/SmallFinalizableHeapBlock.cpp +++ b/lib/Common/Memory/SmallFinalizableHeapBlock.cpp @@ -29,6 +29,30 @@ SmallFinalizableWithBarrierHeapBlockT::Delete(SmallFinalizable } #endif +#ifdef RECYCLER_VISITED_HOST +template +SmallRecyclerVisitedHostHeapBlockT* +SmallRecyclerVisitedHostHeapBlockT::New(HeapBucketT> * bucket) +{ + CompileAssert(TBlockAttributes::MaxObjectSize <= USHRT_MAX); + Assert(bucket->sizeCat <= TBlockAttributes::MaxObjectSize); + Assert((TBlockAttributes::PageCount * AutoSystemInfo::PageSize) / bucket->sizeCat <= USHRT_MAX); + + ushort objectSize = (ushort)bucket->sizeCat; + ushort objectCount = (ushort)(TBlockAttributes::PageCount * AutoSystemInfo::PageSize) / objectSize; + return NoMemProtectHeapNewNoThrowPlusPrefixZ(Base::GetAllocPlusSize(objectCount), SmallRecyclerVisitedHostHeapBlockT, bucket, objectSize, objectCount); +} + +template +void +SmallRecyclerVisitedHostHeapBlockT::Delete(SmallRecyclerVisitedHostHeapBlockT* heapBlock) +{ + Assert(heapBlock->IsRecyclerVisitedHostBlock()); + + NoMemProtectHeapDeletePlusPrefix(Base::GetAllocPlusSize(heapBlock->objectCount), heapBlock); +} +#endif + template SmallFinalizableHeapBlockT * SmallFinalizableHeapBlockT::New(HeapBucketT> * bucket) @@ -74,6 +98,20 @@ SmallFinalizableHeapBlockT::SmallFinalizableHea Assert(!this->isPendingDispose); } +#ifdef RECYCLER_VISITED_HOST +template +SmallFinalizableHeapBlockT::SmallFinalizableHeapBlockT(HeapBucketT> * bucket, ushort objectSize, ushort objectCount, HeapBlockType blockType) + : SmallNormalHeapBlockT(bucket, objectSize, objectCount, blockType) +{ + // We used AllocZ + Assert(this->finalizeCount == 0); + Assert(this->pendingDisposeCount == 0); + Assert(this->disposedObjectList == nullptr); + Assert(this->disposedObjectListTail == nullptr); + Assert(!this->isPendingDispose); +} +#endif + #ifdef RECYCLER_WRITE_BARRIER template SmallFinalizableHeapBlockT::SmallFinalizableHeapBlockT(HeapBucketT> * bucket, ushort objectSize, ushort objectCount, HeapBlockType blockType) @@ -103,6 +141,146 @@ SmallFinalizableHeapBlockT::SetAttributes(void * address, unsi #endif } +#ifdef RECYCLER_VISITED_HOST +template +void +SmallRecyclerVisitedHostHeapBlockT::SetAttributes(void * address, unsigned char attributes) +{ + // Don't call __super, since that has behavior we don't want (it asserts that FinalizeBit is set + // but recycler visited block allows traced only objects; it also unconditionally bumps the heap info + // live/new finalizable object counts which will become unbalance if FinalizeBit is not set). + // We do want the grandparent class behavior though, which actually sets the ObjectInfo bits. + SmallFinalizableHeapBlockT::Base::SetAttributes(address, attributes); + +#ifdef RECYCLER_FINALIZE_CHECK + if (attributes & FinalizeBit) + { + finalizeCount++; + HeapInfo * heapInfo = this->heapBucket->heapInfo; + heapInfo->liveFinalizableObjectCount++; + heapInfo->newFinalizableObjectCount++; + } +#endif +} + +template +template +_NOINLINE +void SmallRecyclerVisitedHostHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext) +{ + unsigned char * attributes = nullptr; + if (!this->TryGetAddressOfAttributes(objectAddress, &attributes)) + { + return; + } + + if (!this->template UpdateAttributesOfMarkedObjects(markContext, objectAddress, this->objectSize, *attributes, + [&](unsigned char _attributes) { *attributes = _attributes; })) + { + // Couldn't mark children- bail out and come back later + this->SetNeedOOMRescan(markContext->GetRecycler()); + } +} + +template +template +bool SmallRecyclerVisitedHostHeapBlockT::UpdateAttributesOfMarkedObjects(MarkContext * markContext, void * objectAddress, size_t objectSize, unsigned char attributes, Fn fn) +{ + bool noOOMDuringMark = true; + + if (attributes & TrackBit) + { + Assert((attributes & LeafBit) == 0); + IRecyclerVisitedObject* recyclerVisited = static_cast(objectAddress); + noOOMDuringMark = markContext->AddPreciselyTracedObject(recyclerVisited); + + if (noOOMDuringMark) + { + // Object has been successfully processed, so clear NewTrackBit + attributes &= ~NewTrackBit; + } + else + { + // Set the NewTrackBit, so that the main thread will redo tracking + attributes |= NewTrackBit; + noOOMDuringMark = false; + } + fn(attributes); + } + +#ifdef RECYCLER_STATS + RECYCLER_STATS_INTERLOCKED_INC(markContext->GetRecycler(), markData.markCount); + RECYCLER_STATS_INTERLOCKED_ADD(markContext->GetRecycler(), markData.markBytes, objectSize); + + // Count track or finalize if we don't have to process it in thread because of OOM. + if ((attributes & (TrackBit | NewTrackBit)) != (TrackBit | NewTrackBit)) + { + // Only count those we have queued, so we don't double count + if (attributes & TrackBit) + { + RECYCLER_STATS_INTERLOCKED_INC(markContext->GetRecycler(), trackCount); + } + if (attributes & FinalizeBit) + { + // we counted the finalizable object here, + // turn off the new bit so we don't count it again + // on Rescan + attributes &= ~NewFinalizeBit; + fn(attributes); + RECYCLER_STATS_INTERLOCKED_INC(markContext->GetRecycler(), finalizeCount); + } + } +#endif + + return noOOMDuringMark; +} + +// static +template +bool +SmallRecyclerVisitedHostHeapBlockT::RescanObject(SmallRecyclerVisitedHostHeapBlockT * block, __in_ecount(localObjectSize) char * objectAddress, uint localObjectSize, + uint objectIndex, Recycler * recycler) +{ + unsigned char const attributes = block->ObjectInfo(objectIndex); + + if ((attributes & TrackBit) != 0) + { + Assert((attributes & LeafBit) == 0); + Assert(block->GetAddressIndex(objectAddress) != SmallHeapBlockT::InvalidAddressBit); + + if (!recycler->AddPreciselyTracedMark(reinterpret_cast(objectAddress))) + { + // Failed to add to the mark stack due to OOM. + return false; + } + + // We have processed this object as tracked, we can clear the NewTrackBit + block->ObjectInfo(objectIndex) &= ~NewTrackBit; + RECYCLER_STATS_INC(recycler, trackCount); + + RECYCLER_STATS_INC(recycler, markData.rescanObjectCount); + RECYCLER_STATS_ADD(recycler, markData.rescanObjectByteCount, localObjectSize); + } + +#ifdef RECYCLER_STATS + if (attributes & FinalizeBit) + { + // Concurrent thread mark the object before the attribute is set and missed the finalize count + // For finalized object, we will always write a dummy vtable before returning to the call, + // so the page will always need to be rescanned, and we can count those here. + + // NewFinalizeBit is cleared if the background thread has already counted the object. + // So if it is still set here, we need to count it + + RECYCLER_STATS_INC_IF(attributes & NewFinalizeBit, recycler, finalizeCount); + block->ObjectInfo(objectIndex) &= ~NewFinalizeBit; + } +#endif + + return true; +} +#endif + template bool SmallFinalizableHeapBlockT::TryGetAttributes(void* objectAddress, unsigned char * pAttr) @@ -476,6 +654,14 @@ namespace Memory template class SmallFinalizableHeapBlockT; template void SmallFinalizableHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext);; template void SmallFinalizableHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext);; +#ifdef RECYCLER_VISITED_HOST + template class SmallRecyclerVisitedHostHeapBlockT; + template void SmallRecyclerVisitedHostHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext); + template void SmallRecyclerVisitedHostHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext); + template class SmallRecyclerVisitedHostHeapBlockT; + template void SmallRecyclerVisitedHostHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext);; + template void SmallRecyclerVisitedHostHeapBlockT::ProcessMarkedObject(void* objectAddress, MarkContext * markContext);; +#endif #ifdef RECYCLER_WRITE_BARRIER template class SmallFinalizableWithBarrierHeapBlockT; diff --git a/lib/Common/Memory/SmallFinalizableHeapBlock.h b/lib/Common/Memory/SmallFinalizableHeapBlock.h index 5669b8c04f0..06697d66ee4 100644 --- a/lib/Common/Memory/SmallFinalizableHeapBlock.h +++ b/lib/Common/Memory/SmallFinalizableHeapBlock.h @@ -5,6 +5,9 @@ namespace Memory { template class SmallFinalizableHeapBucketT; +#ifdef RECYCLER_VISITED_HOST +template class SmallRecyclerVisitedHostHeapBlockT; +#endif #ifdef RECYCLER_WRITE_BARRIER template class SmallFinalizableWithBarrierHeapBlockT; #endif @@ -115,6 +118,9 @@ class SmallFinalizableHeapBlockT : public SmallNormalHeapBlockT * bucket, ushort objectSize, ushort objectCount); +#ifdef RECYCLER_VISITED_HOST + SmallFinalizableHeapBlockT(HeapBucketT> * bucket, ushort objectSize, ushort objectCount, HeapBlockType blockType); +#endif #ifdef RECYCLER_WRITE_BARRIER SmallFinalizableHeapBlockT(HeapBucketT> * bucket, ushort objectSize, ushort objectCount, HeapBlockType blockType); #endif @@ -134,6 +140,48 @@ class SmallFinalizableHeapBlockT : public SmallNormalHeapBlockT +class SmallRecyclerVisitedHostHeapBlockT : public SmallFinalizableHeapBlockT +{ + typedef SmallFinalizableHeapBlockT Base; + friend class HeapBucketT; +public: + typedef TBlockAttributes HeapBlockAttributes; + + static const ObjectInfoBits RequiredAttributes = RecyclerVisitedHostBit; + + static SmallRecyclerVisitedHostHeapBlockT * New(HeapBucketT * bucket); + static void Delete(SmallRecyclerVisitedHostHeapBlockT * block); + + void SetAttributes(void * address, unsigned char attributes); + + template + void ProcessMarkedObject(void* candidate, MarkContext * markContext); + + template + bool UpdateAttributesOfMarkedObjects(MarkContext * markContext, void * objectAddress, size_t objectSize, unsigned char attributes, Fn fn); + + static bool RescanObject(SmallRecyclerVisitedHostHeapBlockT * block, __in_ecount(localObjectSize) char * objectAddress, uint localObjectSize, uint objectIndex, Recycler * recycler); + + SmallRecyclerVisitedHostHeapBlockT * GetNextBlock() const + { + HeapBlock* block = SmallHeapBlockT::GetNextBlock(); + return block ? block->template AsRecyclerVisitedHostBlock() : nullptr; + } + + virtual bool FindHeapObject(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject) override sealed + { + return this->template FindHeapObjectImpl>(objectAddress, recycler, flags, heapObject); + } +protected: + SmallRecyclerVisitedHostHeapBlockT(HeapBucketT * bucket, ushort objectSize, ushort objectCount) + : SmallFinalizableHeapBlockT(bucket, objectSize, objectCount, TBlockAttributes::IsSmallBlock ? Base::SmallRecyclerVisitedHostBlockType : Base::MediumRecyclerVisitedHostBlockType) + { + } +}; +#endif + #ifdef RECYCLER_WRITE_BARRIER template class SmallFinalizableWithBarrierHeapBlockT : public SmallFinalizableHeapBlockT @@ -144,7 +192,6 @@ class SmallFinalizableWithBarrierHeapBlockT : public SmallFinalizableHeapBlockT< typedef TBlockAttributes HeapBlockAttributes; static const ObjectInfoBits RequiredAttributes = FinalizableWithBarrierBit; - static const bool IsLeafOnly = false; static SmallFinalizableWithBarrierHeapBlockT * New(HeapBucketT * bucket); static void Delete(SmallFinalizableWithBarrierHeapBlockT * block); @@ -170,6 +217,11 @@ class SmallFinalizableWithBarrierHeapBlockT : public SmallFinalizableHeapBlockT< typedef SmallFinalizableHeapBlockT SmallFinalizableHeapBlock; typedef SmallFinalizableHeapBlockT MediumFinalizableHeapBlock; +#ifdef RECYCLER_VISITED_HOST +typedef SmallRecyclerVisitedHostHeapBlockT SmallRecyclerVisitedHostHeapBlock; +typedef SmallRecyclerVisitedHostHeapBlockT MediumRecyclerVisitedHostHeapBlock; +#endif + #ifdef RECYCLER_WRITE_BARRIER typedef SmallFinalizableWithBarrierHeapBlockT SmallFinalizableWithBarrierHeapBlock; typedef SmallFinalizableWithBarrierHeapBlockT MediumFinalizableWithBarrierHeapBlock; diff --git a/lib/Common/Memory/SmallFinalizableHeapBucket.cpp b/lib/Common/Memory/SmallFinalizableHeapBucket.cpp index f1a8ecd4b19..66bfa9a5e0b 100644 --- a/lib/Common/Memory/SmallFinalizableHeapBucket.cpp +++ b/lib/Common/Memory/SmallFinalizableHeapBucket.cpp @@ -239,6 +239,10 @@ SmallFinalizableHeapBucketBaseT::VerifyMark() namespace Memory { template class SmallFinalizableHeapBucketBaseT; +#ifdef RECYCLER_VISITED_HOST + template class SmallFinalizableHeapBucketBaseT; + template class SmallFinalizableHeapBucketBaseT; +#endif #ifdef RECYCLER_WRITE_BARRIER template class SmallFinalizableHeapBucketBaseT; #endif diff --git a/lib/Common/Memory/SmallFinalizableHeapBucket.h b/lib/Common/Memory/SmallFinalizableHeapBucket.h index dd2bf1bc49b..5fd703bd782 100644 --- a/lib/Common/Memory/SmallFinalizableHeapBucket.h +++ b/lib/Common/Memory/SmallFinalizableHeapBucket.h @@ -61,6 +61,12 @@ template class SmallFinalizableHeapBucketT : public SmallFinalizableHeapBucketBaseT > { }; +#ifdef RECYCLER_VISITED_HOST +template +class SmallRecyclerVisitedHostHeapBucketT : public SmallFinalizableHeapBucketBaseT > +{ +}; +#endif #ifdef RECYCLER_WRITE_BARRIER template class SmallFinalizableWithBarrierHeapBucketT : public SmallFinalizableHeapBucketBaseT > @@ -70,6 +76,10 @@ class SmallFinalizableWithBarrierHeapBucketT : public SmallFinalizableHeapBucket typedef SmallFinalizableHeapBucketT MediumFinalizableHeapBucket; typedef SmallFinalizableHeapBucketT SmallFinalizableHeapBucket; +#ifdef RECYCLER_VISITED_HOST +typedef SmallRecyclerVisitedHostHeapBucketT MediumRecyclerVisitedHostHeapBucket; +typedef SmallRecyclerVisitedHostHeapBucketT SmallRecyclerVisitedHostHeapBucket; +#endif #ifdef RECYCLER_WRITE_BARRIER typedef SmallFinalizableWithBarrierHeapBucketT MediumFinalizableWithBarrierHeapBucket; typedef SmallFinalizableWithBarrierHeapBucketT SmallFinalizableWithBarrierHeapBucket; @@ -81,11 +91,42 @@ template class SmallHeapBlockType { public: - CompileAssert(attributes & FinalizeBit); + + CompileAssert((attributes & FinalizeBit) != 0); +#ifdef RECYCLER_VISITED_HOST + // attributes with RecyclerVisitedHostBit must use SmallRecyclerVisitedHostHeap{Bucket|Block}T + CompileAssert((attributes & RecyclerVisitedHostBit) == 0); +#endif typedef SmallFinalizableHeapBlockT BlockType; typedef SmallFinalizableHeapBucketT BucketType; }; +#ifdef RECYCLER_VISITED_HOST +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostBit), SmallAllocationBlockAttributes> +{ +public: + typedef SmallRecyclerVisitedHostHeapBlock BlockType; + typedef SmallRecyclerVisitedHostHeapBucket BucketType; +}; + +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostTracedFinalizableBlockTypeBits), SmallAllocationBlockAttributes> +{ +public: + typedef SmallRecyclerVisitedHostHeapBlock BlockType; + typedef SmallRecyclerVisitedHostHeapBucket BucketType; +}; + +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostFinalizableBlockTypeBits), SmallAllocationBlockAttributes> +{ +public: + typedef SmallRecyclerVisitedHostHeapBlock BlockType; + typedef SmallRecyclerVisitedHostHeapBucket BucketType; +}; +#endif + template <> class SmallHeapBlockType { @@ -137,6 +178,32 @@ class SmallHeapBlockType typedef MediumFinalizableHeapBucket BucketType; }; +#ifdef RECYCLER_VISITED_HOST +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostBit), MediumAllocationBlockAttributes> +{ +public: + typedef MediumRecyclerVisitedHostHeapBlock BlockType; + typedef MediumRecyclerVisitedHostHeapBucket BucketType; +}; + +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostTracedFinalizableBlockTypeBits), MediumAllocationBlockAttributes> +{ +public: + typedef MediumRecyclerVisitedHostHeapBlock BlockType; + typedef MediumRecyclerVisitedHostHeapBucket BucketType; +}; + +template <> +class SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostFinalizableBlockTypeBits), MediumAllocationBlockAttributes> +{ +public: + typedef MediumRecyclerVisitedHostHeapBlock BlockType; + typedef MediumRecyclerVisitedHostHeapBucket BucketType; +}; +#endif + template <> class SmallHeapBlockType { @@ -229,6 +296,41 @@ class HeapBucketGroup } }; +#ifdef RECYCLER_VISITED_HOST + template <> + class BucketGetter<(ObjectInfoBits)(RecyclerVisitedHostBit)> + { + public: + typedef typename SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostBit), TBlockAttributes>::BucketType BucketType; + static BucketType& GetBucket(HeapBucketGroup * HeapBucketGroup) + { + return HeapBucketGroup->recyclerVisitedHostHeapBucket; + } + }; + + template <> + class BucketGetter<(ObjectInfoBits)(RecyclerVisitedHostTracedFinalizableBlockTypeBits)> + { + public: + typedef typename SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostTracedFinalizableBlockTypeBits), TBlockAttributes>::BucketType BucketType; + static BucketType& GetBucket(HeapBucketGroup * HeapBucketGroup) + { + return HeapBucketGroup->recyclerVisitedHostHeapBucket; + } + }; + + template <> + class BucketGetter<(ObjectInfoBits)(RecyclerVisitedHostFinalizableBlockTypeBits)> + { + public: + typedef typename SmallHeapBlockType<(ObjectInfoBits)(RecyclerVisitedHostFinalizableBlockTypeBits), TBlockAttributes>::BucketType BucketType; + static BucketType& GetBucket(HeapBucketGroup * HeapBucketGroup) + { + return HeapBucketGroup->recyclerVisitedHostHeapBucket; + } + }; +#endif + #ifdef RECYCLER_WRITE_BARRIER template <> class BucketGetter @@ -323,6 +425,9 @@ class HeapBucketGroup SmallNormalHeapBucketT heapBucket; SmallLeafHeapBucketT leafHeapBucket; SmallFinalizableHeapBucketT finalizableHeapBucket; +#ifdef RECYCLER_VISITED_HOST + SmallRecyclerVisitedHostHeapBucketT recyclerVisitedHostHeapBucket; +#endif #ifdef RECYCLER_WRITE_BARRIER SmallNormalWithBarrierHeapBucketT smallNormalWithBarrierHeapBucket; SmallFinalizableWithBarrierHeapBucketT smallFinalizableWithBarrierHeapBucket; diff --git a/lib/Common/Memory/SmallNormalHeapBlock.h b/lib/Common/Memory/SmallNormalHeapBlock.h index b9137aeefaa..c6f9edaace2 100644 --- a/lib/Common/Memory/SmallNormalHeapBlock.h +++ b/lib/Common/Memory/SmallNormalHeapBlock.h @@ -18,7 +18,6 @@ class SmallNormalHeapBlockT : public SmallHeapBlockT typedef typename Base::SmallHeapBlockBitVector SmallHeapBlockBitVector; typedef TBlockAttributes HeapBlockAttributes; static const ObjectInfoBits RequiredAttributes = NoBit; - static const bool IsLeafOnly = false; static SmallNormalHeapBlockT * New(HeapBucketT * bucket); static void Delete(SmallNormalHeapBlockT * block); @@ -67,7 +66,6 @@ class SmallNormalWithBarrierHeapBlockT : public SmallNormalHeapBlockT * bucket); static void Delete(SmallNormalWithBarrierHeapBlockT * heapBlock); diff --git a/lib/Common/Memory/SmallNormalHeapBucket.cpp b/lib/Common/Memory/SmallNormalHeapBucket.cpp index 4113e1a46b2..150be744ddb 100644 --- a/lib/Common/Memory/SmallNormalHeapBucket.cpp +++ b/lib/Common/Memory/SmallNormalHeapBucket.cpp @@ -683,6 +683,11 @@ namespace Memory template class SmallNormalHeapBucketBase; template class SmallNormalHeapBucketBase; +#ifdef RECYCLER_VISITED_HOST + template class SmallNormalHeapBucketBase; + template class SmallNormalHeapBucketBase; +#endif + #ifdef RECYCLER_WRITE_BARRIER template class SmallNormalHeapBucketBase; template class SmallNormalHeapBucketBase; diff --git a/lib/Parser/RegexParser.cpp b/lib/Parser/RegexParser.cpp index 132bd7ccd2c..c3776c654e1 100644 --- a/lib/Parser/RegexParser.cpp +++ b/lib/Parser/RegexParser.cpp @@ -769,11 +769,11 @@ namespace UnifiedRegex last->head = revisedPrev; nextList = nextList->tail; } - AnalysisAssert(nextList != nullptr); if (last == 0) node = Anew(ctAllocator, AltNode, node, nextList); else last->tail = nextList; + AnalysisAssert(nextList != nullptr); while (nextList->tail != 0) nextList = nextList->tail; last = nextList;