diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 40732a494c444bb2af87d9389e88000e516cbb55..5ddf1abab32d32d1a012294d8ecd36162d34a1ee 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -82,6 +82,8 @@ private: // region info is stored in the metadata of its primary unit (i.e. the first unit). class RegionDesc { public: + static constexpr HeapAddress TEMPORARY_PTR = 0x1234; + enum RouteState : uint8_t { INITIAL = 0, COPYABLE, @@ -140,7 +142,7 @@ public: RegionBitmap *GetMarkBitmap() { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.markBitmap_, std::memory_order_acquire); - if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { + if (reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR) { return nullptr; } return bitmap; @@ -150,13 +152,13 @@ public: { do { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.markBitmap_, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { + if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + RegionBitmap *newValue = reinterpret_cast(RegionDesc::TEMPORARY_PTR); if (__atomic_compare_exchange_n(&metadata.liveInfo_.markBitmap_, &bitmap, newValue, false, std::memory_order_seq_cst, std::memory_order_relaxed)) { RegionBitmap *allocated = @@ -174,7 +176,7 @@ public: RegionBitmap *GetResurrectBitmap() { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.resurrectBitmap_, std::memory_order_acquire); - if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { + if (reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR) { return nullptr; } return bitmap; @@ -184,13 +186,13 @@ public: { do { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.resurrectBitmap_, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { + if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + RegionBitmap *newValue = reinterpret_cast(RegionDesc::TEMPORARY_PTR); if (__atomic_compare_exchange_n(&metadata.liveInfo_.resurrectBitmap_, &bitmap, newValue, false, std::memory_order_seq_cst, std::memory_order_relaxed)) { RegionBitmap *allocated = @@ -209,7 +211,7 @@ public: RegionBitmap* GetEnqueueBitmap() { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { + if (reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR) { return nullptr; } return bitmap; @@ -219,13 +221,13 @@ public: { do { RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { + if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + RegionBitmap* newValue = reinterpret_cast(RegionDesc::TEMPORARY_PTR); if (__atomic_compare_exchange_n(&metadata.liveInfo_.enqueueBitmap_, &bitmap, newValue, false, std::memory_order_seq_cst, std::memory_order_relaxed)) { RegionBitmap *allocated = @@ -1026,8 +1028,6 @@ private: class RegionLiveDesc { public: - static constexpr HeapAddress TEMPORARY_PTR = 0x1234; - void Init(RegionDesc *region) { relatedRegion_ = region; @@ -1221,13 +1221,20 @@ public: } explicit InlinedRegionMetaData(RegionDesc *regionDesc) - : regionDesc_(regionDesc), regionRSet_(regionDesc->GetRSet()), regionType_(regionDesc->GetRegionType()) + : regionDesc_(regionDesc), regionRSet_(regionDesc->GetRSet()), regionType_(regionDesc->GetRegionType()), + traceLine_(®ionDesc->metadata.traceLine), + liveInfo_markBitmap_(®ionDesc->metadata.liveInfo_.markBitmap_) { // Since this is a backup copy of `RegionDesc`, create rset at first to guarantee data consistency DCHECK_CC(regionRSet_ != nullptr); // Not insert to regionList and reset regionType yet DCHECK_CC(regionType_ == RegionType::FREE_REGION); DCHECK_CC(regionType_ == regionDesc_->GetRegionType()); + DCHECK_CC(traceLine_ == ®ionDesc->metadata.traceLine); + DCHECK_CC(liveInfo_markBitmap_ == ®ionDesc->metadata.liveInfo_.markBitmap_); + // make compiler happy in release mode + (void)traceLine_; + (void)liveInfo_markBitmap_; } ~InlinedRegionMetaData() = default; @@ -1321,6 +1328,8 @@ public: RegionRSet *regionRSet_ {nullptr}; RegionType regionType_ {}; // fixme: inline more + uintptr_t *traceLine_ {nullptr}; + RegionBitmap **liveInfo_markBitmap_ {nullptr}; friend class RegionDesc; }; @@ -1330,6 +1339,18 @@ public: static constexpr size_t UNIT_HEADER_SIZE = AlignUp(sizeof(InlinedRegionMetaData), UNIT_BEGIN_ALIGN); // default common region unit available size. static constexpr size_t UNIT_AVAILABLE_SIZE = UNIT_SIZE - UNIT_HEADER_SIZE; + // Another definition for IR. + static constexpr uint8_t LARGE_SIZED_UNITS_IN_REGION_DESC = static_cast(UnitRole::LARGE_SIZED_UNITS); + + enum RegionBitMask : uint32_t { + BIT_MASK_REGION_TYPE = 0x00000001, + BIT_MASK_MARKED_REGION = 0x00000020, + BIT_MASK_ENQUEUED_REGION = 0x00000040, + BIT_MASK_RESURRECTED_REGION = 0x00000080, + BIT_MASK_FIXED_REGION = 0x00000100, + BIT_MASK_REGION_CELLCOUNT = 0x00000200, + BIT_MASK_IS_JITFORT_AWAIT_INSTALL = 0x00010000, + }; private: @@ -1405,6 +1426,11 @@ public: friend constexpr size_t GetMetaDataInRegionOffset(); static constexpr size_t REGION_RSET_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionRSet_); static constexpr size_t REGION_TYPE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionType_); + static constexpr size_t TRACE_LINE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, traceLine_); + static constexpr size_t LIVEINFO_MARKBITMAP_LINE_IN_INLINED_METADATA_OFFSET = + MEMBER_OFFSET(InlinedRegionMetaData, liveInfo_markBitmap_); + static constexpr size_t UNITROLE_IN_UNITMETADATA_OFFSET = MEMBER_OFFSET(struct UnitMetadata, unitBits); + static constexpr size_t REGIONBITS_IN_UNITMETADATA_OFFSET = MEMBER_OFFSET(struct UnitMetadata, regionBits); }; HeapAddress RegionDesc::InlinedRegionMetaData::GetRegionStart() const diff --git a/common_components/heap/collector/region_bitmap.h b/common_components/heap/collector/region_bitmap.h index 0aeac4f6f4ed0a7e8a72ae17ac356b9ad4b2931b..218c4ffa638162219f598be870735a1bc1e5452b 100755 --- a/common_components/heap/collector/region_bitmap.h +++ b/common_components/heap/collector/region_bitmap.h @@ -72,6 +72,7 @@ struct RegionBitmap { return ret; } }; +static constexpr size_t MARKWORDS_IN_REGIONBITMAP_OFFSET = MEMBER_OFFSET(struct RegionBitmap, markWords); } // namespace common #endif // COMMON_COMPONENTS_HEAP_COLLECTOR_REGION_BITMAP_H diff --git a/common_components/heap/heap.h b/common_components/heap/heap.h index 58d69eab3335cd8a3edadbccd263b3ef461a5c6d..a2aba9d89ef4bd6339edd3561b88b7d0d349aa67 100755 --- a/common_components/heap/heap.h +++ b/common_components/heap/heap.h @@ -39,7 +39,7 @@ class Heap { public: // These need to keep same with that in `RegionDesc` static constexpr size_t NORMAL_UNIT_SIZE = 256 * 1024; - static constexpr size_t NORMAL_UNIT_HEADER_SIZE = AlignUp(2 * sizeof(void *) + sizeof(uint8_t), 8); + static constexpr size_t NORMAL_UNIT_HEADER_SIZE = AlignUp(4 * sizeof(void *) + sizeof(uint8_t), 8); static constexpr size_t NORMAL_UNIT_AVAILABLE_SIZE = NORMAL_UNIT_SIZE - NORMAL_UNIT_HEADER_SIZE; static constexpr size_t GetNormalRegionSize() diff --git a/common_components/mutator/mutator.h b/common_components/mutator/mutator.h index f5a4950dc0d69356c489d90145df3521d1b85b7a..145db24a6293ba7c39a0f53821c990c805e821c6 100755 --- a/common_components/mutator/mutator.h +++ b/common_components/mutator/mutator.h @@ -259,6 +259,7 @@ public: #endif NO_INLINE_CC void RememberObjectInSatbBuffer(const BaseObject* obj) { RememberObjectImpl(obj); } + NO_INLINE_CC void RememberObjectInSatbBufferSimply(const BaseObject* obj) { RememberObjectImplSimply(obj); } const SatbBuffer::TreapNode* GetSatbBufferNode() const { return satbNode_; } @@ -319,6 +320,13 @@ private: } } } + void RememberObjectImplSimply(const BaseObject* obj) + { + if (SatbBuffer::Instance().ShouldEnqueueSimply(obj)) { + SatbBuffer::Instance().EnsureGoodNode(satbNode_); + satbNode_->Push(obj); + } + } // Indicate the current mutator phase and use which barrier in concurrent gc // ATTENTION: THE LAYOUT FOR GCPHASE MUST NOT BE CHANGED! // currently store struct instead of pointer to keep GCPHASE offset to this is 0 diff --git a/common_components/mutator/satb_buffer.cpp b/common_components/mutator/satb_buffer.cpp index cf3eddbc46e2528e00fb2d18580c9c3f18b9b7b5..3d51f01965fe1aaf63a65ffea76f55c5b4d1be8e 100755 --- a/common_components/mutator/satb_buffer.cpp +++ b/common_components/mutator/satb_buffer.cpp @@ -39,4 +39,9 @@ bool SatbBuffer::ShouldEnqueue(const BaseObject* obj) } return !RegionSpace::EnqueueObject(obj); } + +bool SatbBuffer::ShouldEnqueueSimply(const BaseObject* obj) +{ + return !RegionSpace::EnqueueObject(obj); +} } // namespace common diff --git a/common_components/mutator/satb_buffer.h b/common_components/mutator/satb_buffer.h index 10783b4be9b86f031b3471bb9366529edbd61e6f..d11389050c34aea2c9394bea10d99d724763fa71 100755 --- a/common_components/mutator/satb_buffer.h +++ b/common_components/mutator/satb_buffer.h @@ -209,6 +209,7 @@ public: } } bool ShouldEnqueue(const BaseObject* obj); + bool ShouldEnqueueSimply(const BaseObject* obj); // must not have thread racing void Init() diff --git a/ecmascript/compiler/stub_builder.cpp b/ecmascript/compiler/stub_builder.cpp index 2a39f1a1177aad487a1bb20e89b87b1d252f851d..0ddad3b89cd85b75f4181ffc6c0714b3eccab5ec 100644 --- a/ecmascript/compiler/stub_builder.cpp +++ b/ecmascript/compiler/stub_builder.cpp @@ -1924,6 +1924,17 @@ GateRef StubBuilder::GetCMCRegionType(GateRef obj) return regionDescAddr; } +GateRef StubBuilder::CMCIsInYoungSpaceForWB(GateRef regionType) +{ + auto env = GetEnvironment(); + GateRef ret = LogicOrBuilder(env) + .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::THREAD_LOCAL_REGION)))) + .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::RECENT_FULL_REGION)))) + .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::FROM_REGION)))) + .Done(); + return ret; +} + GateRef StubBuilder::CMCIsInYoungSpace(GateRef regionType) { auto env = GetEnvironment(); @@ -1931,6 +1942,7 @@ GateRef StubBuilder::CMCIsInYoungSpace(GateRef regionType) .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::THREAD_LOCAL_REGION)))) .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::RECENT_FULL_REGION)))) .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::FROM_REGION)))) + .Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::EXEMPTED_FROM_REGION)))) .Done(); return ret; } @@ -1939,8 +1951,8 @@ GateRef StubBuilder::IsOldToYoung(GateRef objRegionType, GateRef valueRegionType { auto env = GetEnvironment(); GateRef isOldToYoung = LogicAndBuilder(env) - .And(BoolNot(CMCIsInYoungSpace(objRegionType))) - .And(CMCIsInYoungSpace(valueRegionType)).Done(); + .And(BoolNot(CMCIsInYoungSpaceForWB(objRegionType))) + .And(CMCIsInYoungSpaceForWB(valueRegionType)).Done(); return isOldToYoung; } @@ -2035,6 +2047,128 @@ GateRef StubBuilder::ShouldUpdateRememberSet(GateRef glue, GateRef gcPhase) return ret; } +GateRef StubBuilder::IsNewObjectSinceTrace(GateRef obj) +{ + GateRef inlineRegionMetaDataPtr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + GateRef traceLinePtr = LoadPrimitive(VariableType::NATIVE_POINTER(), inlineRegionMetaDataPtr, + IntPtr(common::RegionDesc::TRACE_LINE_IN_INLINED_METADATA_OFFSET)); + GateRef traceLine = LoadZeroOffsetPrimitive(VariableType::NATIVE_POINTER(), traceLinePtr); + return Int64UnsignedLessThanOrEqual(traceLine, TaggedCastToIntPtr(obj)); +} + +GateRef StubBuilder::IsLargeRegion(GateRef regionDescPtr) +{ + GateRef unitRole = LoadPrimitive(VariableType::INT8(), regionDescPtr, + IntPtr(common::RegionDesc::UNITROLE_IN_UNITMETADATA_OFFSET)); + return Int8Equal(unitRole, Int8(common::RegionDesc::LARGE_SIZED_UNITS_IN_REGION_DESC)); +} + +GateRef StubBuilder::GetMarkBitmap(GateRef obj) +{ + GateRef inlineRegionMetaDataPtr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + return LoadPrimitive(VariableType::NATIVE_POINTER(), inlineRegionMetaDataPtr, + IntPtr(common::RegionDesc::LIVEINFO_MARKBITMAP_LINE_IN_INLINED_METADATA_OFFSET)); +} + +GateRef StubBuilder::IsMarked(GateRef markBitmapPtr, GateRef start) +{ + GateRef div = IntPtrDiv(start, IntPtr(common::kMarkedBytesPerBit)); + GateRef headWordIdx = IntPtrDiv(div, Int64(common::kBitsPerWord)); + GateRef headMaskBitStart = IntPtrMod(div, Int64(common::kBitsPerWord)); + GateRef mask = Int64LSL(Int64(1), headMaskBitStart); + GateRef headWordOffset = Int64Mul(headWordIdx, Int64(common::kBytesPerWord)); + + GateRef offset = Int64Add(IntPtr(common::MARKWORDS_IN_REGIONBITMAP_OFFSET), headWordOffset); + GateRef words = LoadPrimitive(VariableType::INT64(), markBitmapPtr, offset); + return Int64NotEqual(Int64And(words, mask), Int64(0)); +} + +GateRef StubBuilder::IsMarkedObject(GateRef obj) +{ + auto env = GetEnvironment(); + Label entry(env); + env->SubCfgEntry(&entry); + + Label exit(env); + DEFVARIABLE(result, VariableType::BOOL(), False()); + + Label isLargeRegionTrue(env); + Label isLargeRegionContinue(env); + GateRef inlineRegionMetaDataPtr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + GateRef regionDescPtr = LoadZeroOffsetPrimitive(VariableType::NATIVE_POINTER(), inlineRegionMetaDataPtr); + BRANCH(IsLargeRegion(regionDescPtr), &isLargeRegionTrue, &isLargeRegionContinue); + Bind(&isLargeRegionTrue); + { + GateRef regionBits = LoadPrimitive(VariableType::INT32(), regionDescPtr, + IntPtr(common::RegionDesc::REGIONBITS_IN_UNITMETADATA_OFFSET)); + GateRef regionBitsMask = Int32And(regionBits, + Int32(common::RegionDesc::RegionBitMask::BIT_MASK_MARKED_REGION)); + result = Int32Equal(regionBitsMask, Int32(common::RegionDesc::RegionBitMask::BIT_MASK_MARKED_REGION)); + Jump(&exit); + } + Bind(&isLargeRegionContinue); + + Label markBitmapNotInit(env); + Label checkMark(env); + GateRef markBitmapPtr = GetMarkBitmap(obj); + GateRef markBitmap = LoadZeroOffsetPrimitive(VariableType::NATIVE_POINTER(), markBitmapPtr); + BRANCH(IntPtrEuqal(TaggedCastToIntPtr(markBitmap), IntPtr(common::RegionDesc::TEMPORARY_PTR)), + &exit, &markBitmapNotInit); + Bind(&markBitmapNotInit); + { + BRANCH(IntPtrEuqal(TaggedCastToIntPtr(markBitmap), IntPtr(0)), &exit, &checkMark); + } + Bind(&checkMark); + + GateRef regionBase = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + GateRef objOffset = PtrSub(TaggedCastToIntPtr(obj), regionBase); + result = IsMarked(markBitmap, objOffset); + Jump(&exit); + + Bind(&exit); + auto ret = *result; + env->SubCfgExit(); + return ret; +} + +GateRef StubBuilder::ShouldEnqueue(GateRef glue, GateRef obj) +{ + auto env = GetEnvironment(); + Label entry(env); + env->SubCfgEntry(&entry); + + Label exit(env); + DEFVARIABLE(result, VariableType::BOOL(), False()); + + Label checkYoungSpace(env); + Label checkTrace(env); + BRANCH(Int32Equal(GetGCReason(glue), Int32(common::GCReason::GC_REASON_YOUNG)), &checkYoungSpace, &checkTrace); + Bind(&checkYoungSpace); + { + BRANCH(BoolNot(CMCIsInYoungSpace(GetCMCRegionType(obj))), &exit, &checkTrace); + } + Bind(&checkTrace); + + Label checkMarkObject(env); + BRANCH_UNLIKELY(IsNewObjectSinceTrace(obj), &exit, &checkMarkObject); + Bind(&checkMarkObject); + + Label checkEnqueueObject(env); + BRANCH_UNLIKELY(IsMarkedObject(obj), &exit, &checkEnqueueObject); + Bind(&checkEnqueueObject); + result = True(); + Jump(&exit); + + Bind(&exit); + auto ret = *result; + env->SubCfgExit(); + return ret; +} + void StubBuilder::CMCSetValueWithBarrier(GateRef glue, GateRef obj, [[maybe_unused]]GateRef offset, GateRef value) { auto env = GetEnvironment(); @@ -2061,8 +2195,13 @@ void StubBuilder::CMCSetValueWithBarrier(GateRef glue, GateRef obj, [[maybe_unus BRANCH_UNLIKELY(shouldProcessSATB, &markInBuffer, &exit); Bind(&markInBuffer); { - CallNGCRuntime(glue, RTSTUB_ID(MarkInBuffer), {value}); - Jump(&exit); + Label checkEnqueue(env); + BRANCH_UNLIKELY(ShouldEnqueue(glue, value), &checkEnqueue, &exit); + Bind(&checkEnqueue); + { + CallNGCRuntime(glue, RTSTUB_ID(MarkInBuffer), {value}); + Jump(&exit); + } } Bind(&exit); env->SubCfgExit(); diff --git a/ecmascript/compiler/stub_builder.h b/ecmascript/compiler/stub_builder.h index 6522764427ed2487f3d6480dae0ccde54f5e030e..f42cbb25f94bc15932a5dc554a46a4928500b005 100644 --- a/ecmascript/compiler/stub_builder.h +++ b/ecmascript/compiler/stub_builder.h @@ -823,12 +823,19 @@ public: GateRef GetCMCRegionType(GateRef obj); GateRef GetGCPhase(GateRef glue); GateRef GetGCReason(GateRef glue); + GateRef CMCIsInYoungSpaceForWB(GateRef regionType); GateRef CMCIsInYoungSpace(GateRef regionType); GateRef IsOldToYoung(GateRef objRegionType, GateRef valueRegionType); void MarkRSetCardTable(GateRef obj, Label *exit); GateRef ShouldGetGCReason(GateRef gcPhase); GateRef ShouldProcessSATB(GateRef gcPhase); GateRef ShouldUpdateRememberSet(GateRef glue, GateRef gcPhase); + GateRef IsNewObjectSinceTrace(GateRef obj); + GateRef IsLargeRegion(GateRef regionDescPtr); + GateRef GetMarkBitmap(GateRef obj); + GateRef IsMarked(GateRef markBitmapPtr, GateRef start); + GateRef IsMarkedObject(GateRef obj); + GateRef ShouldEnqueue(GateRef glue, GateRef obj); void CMCSetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value); void CMCArrayCopyWriteBarrier(GateRef glue, GateRef dstObj, GateRef src, GateRef dst, GateRef count); void SetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value,