Add "Evict L1" option to memfence intrinsic and enable it on Release semantics fences

PawelJurek · igcbot · commit 9efd6cb8f164 · 2022-02-22T11:17:21.000+01:00
Normally evicts shouldn't be needed, as by default runtime sets L1 WriteBypass mode. This is an unwritten contract between runtime and compiler. In order for the kernel to be correct, we need to insert L1 Evicts in cases of Release semantics.
diff --git a/IGC/BiFModule/Implementation/IGCBiF_Intrinsics.cl b/IGC/BiFModule/Implementation/IGCBiF_Intrinsics.cl
@@ -103,7 +103,7 @@ local uchar* __builtin_IB_AllocLocalMemPool(bool allocAllWorkgroups, uint numAdd
 
 // Memory fences
 // See GenISAIntrinsics.td for documentation
-void     __builtin_IB_memfence(bool commitEnable, bool flushRW, bool flushConstant, bool flushTexture, bool flushIcache, bool isGlobal, bool invalidateL1);
+void     __builtin_IB_memfence(bool commitEnable, bool flushRW, bool flushConstant, bool flushTexture, bool flushIcache, bool isGlobal, bool invalidateL1, bool evictL1);
 void     __builtin_IB_flush_sampler_cache(void);
 void     __builtin_IB_typedmemfence(bool invalidateCache);
 
diff --git a/IGC/BiFModule/Implementation/atomics.cl b/IGC/BiFModule/Implementation/atomics.cl
@@ -26,7 +26,7 @@ extern __constant int __UseNativeFP64GlobalAtomicAdd;
   __local int* __builtin_IB_get_local_lock();
   __global int* __builtin_IB_get_global_lock();
   void __builtin_IB_eu_thread_pause(uint value);
-  void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1);
+  void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1, bool evictL1);
 
 #define LOCAL_SPINLOCK_START() \
   { \
@@ -56,14 +56,14 @@ extern __constant int __UseNativeFP64GlobalAtomicAdd;
   if( ( (Semantics) & ( SEMANTICS_PRE_OP_NEED_FENCE ) ) > 0 )                         \
   {                                                                                   \
       bool flushL3 = (isGlobal) && ((Scope) == Device || (Scope) == CrossDevice);     \
-      __intel_memfence_handler(flushL3, isGlobal, false);                             \
+      __intel_memfence_handler(flushL3, isGlobal, false, false);                      \
   }
 
 #define FENCE_POST_OP(Scope, Semantics, isGlobal)                                     \
   if( ( (Semantics) & ( SEMANTICS_POST_OP_NEEDS_FENCE ) ) > 0 )                       \
   {                                                                                   \
       bool flushL3 = (isGlobal) && ((Scope) == Device || (Scope) == CrossDevice);     \
-      __intel_memfence_handler(flushL3, isGlobal, false);                             \
+      __intel_memfence_handler(flushL3, isGlobal, false, false);                      \
   }
 
 // This fencing scheme allows us to obey the memory model when coherency is
diff --git a/IGC/BiFModule/Implementation/barrier.cl b/IGC/BiFModule/Implementation/barrier.cl
@@ -18,37 +18,39 @@ extern __constant int __OptDisable;
 
 // MEMFENCE IMPLEMENTATION
 
-void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1)
+void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1, bool evictL1)
 {
-#define MEMFENCE_IF(V1, V5, V6)                                    \
-if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6)         \
-{                                                                  \
-    __builtin_IB_memfence(true, V1, false, false, false, V5, V6);  \
+#define MEMFENCE_IF(V1, V5, V6, V7)                                           \
+if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6 && evictL1 == V7)   \
+{                                                                             \
+    __builtin_IB_memfence(true, V1, false, false, false, V5, V6, V7);         \
 } else
 
 // Generate combinations for all MEMFENCE_IF cases, e.g.:
 // true, true, true
 // true, true, false etc.
+#define MF_L3(...) MF_L2(__VA_ARGS__,false) MF_L2(__VA_ARGS__,true)
 #define MF_L2(...) MF_L1(__VA_ARGS__,false) MF_L1(__VA_ARGS__,true)
 #define MF_L1(...) MEMFENCE_IF(__VA_ARGS__,false) MEMFENCE_IF(__VA_ARGS__,true)
-MF_L2(false)
-MF_L2(true) {}
+MF_L3(false)
+MF_L3(true) {}
 
 #undef MEMFENCE_IF
+#undef MF_L3
 #undef MF_L2
 #undef MF_L1
 }
-void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1)
+void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1, bool evictL1)
 {
-    __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1);
+    __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1, evictL1);
 }
 
-void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1)
+void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1, bool evictL1)
 {
     if (__OptDisable)
-        __intel_memfence_optnone(flushRW, isGlobal, invalidateL1);
+        __intel_memfence_optnone(flushRW, isGlobal, invalidateL1, evictL1);
     else
-        __intel_memfence(flushRW, isGlobal, invalidateL1);
+        __intel_memfence(flushRW, isGlobal, invalidateL1, evictL1);
 }
 
 // TYPEDMEMFENCE IMPLEMENTATION
@@ -81,6 +83,7 @@ static void __intel_atomic_work_item_fence( Scope_t Memory, uint Semantics )
     bool fence = Semantics & ( Acquire | Release | AcquireRelease | SequentiallyConsistent );
 
     bool invalidateL1 = Semantics & ( Acquire | AcquireRelease | SequentiallyConsistent );
+    bool evictL1 = Semantics & ( Release | AcquireRelease | SequentiallyConsistent );
 
     // We always need to 'fence' image memory (aka, flush caches, drain pipelines)
     fence |= ( Semantics & ImageMemory );
@@ -97,12 +100,12 @@ static void __intel_atomic_work_item_fence( Scope_t Memory, uint Semantics )
         // although on some platforms they may be elided; platform-specific checks are performed in codegen
         if (Semantics & WorkgroupMemory)
         {
-           __intel_memfence_handler(false, false, false);
+           __intel_memfence_handler(false, false, false, false);
         }
         if (Semantics & CrossWorkgroupMemory)
         {
            bool flushL3 = Memory == Device || Memory == CrossDevice;
-           __intel_memfence_handler(flushL3, true, invalidateL1);
+           __intel_memfence_handler(flushL3, true, invalidateL1, evictL1);
         }
     }
 }
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp
@@ -15551,14 +15551,19 @@ void EmitPass::emitThreadGroupBarrier(llvm::Instruction* inst)
     }
 }
 
-LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1) const
+LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1, bool EvictL1) const
 {
     LSC_FENCE_OP op = LSC_FENCE_OP_NONE;
     if (InvalidateL1 || (IsGlobalMemFence && m_currShader->m_Platform->getWATable().Wa_14012437816))
     {
         op = LSC_FENCE_OP_INVALIDATE;
     }
 
+    if (EvictL1)
+    {
+        op = LSC_FENCE_OP_EVICT;
+    }
+
     // For experiment on XeHP SDV
     if (op == LSC_FENCE_OP_NONE && IsGlobalMemFence && IGC_IS_FLAG_ENABLED(EnableL3FlushForGlobal))
     {
@@ -15569,7 +15574,7 @@ LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool Invalidat
 
 void EmitPass::emitMemoryFence(llvm::Instruction* inst)
 {
-    static constexpr int ExpectedNumberOfArguments = 7;
+    static constexpr int ExpectedNumberOfArguments = 8;
     IGC_ASSERT(cast<CallInst>(inst)->getNumArgOperands() == ExpectedNumberOfArguments);
     CodeGenContext* ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
 
@@ -15583,6 +15588,9 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
     bool L3_Flush_Instructions = true;
     bool Global_Mem_Fence = true;
     bool L1_Invalidate = ctx->platform.hasL1ReadOnlyCache();
+    // Note: this flag is respected only for LSC case currently.
+    // TODO: add support for non-LSC and typed fence.
+    bool L1_Evict = true;
 
     std::array<reference_wrapper<bool>, ExpectedNumberOfArguments> MemFenceArguments{
       CommitEnable,
@@ -15592,6 +15600,7 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
       L3_Flush_Instructions,
       Global_Mem_Fence,
       L1_Invalidate,
+      L1_Evict
     };
 
     for (size_t i = 0; i < MemFenceArguments.size(); ++i) {
@@ -15639,7 +15648,7 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
         {
             scope = LSC_SCOPE_TILE;
         }
-        LSC_FENCE_OP op = getLSCMemoryFenceOp(Global_Mem_Fence, L1_Invalidate);
+        LSC_FENCE_OP op = getLSCMemoryFenceOp(Global_Mem_Fence, L1_Invalidate, L1_Evict);
         if (inst->getMetadata("forceFlushNone"))
         {
             op = LSC_FENCE_OP_NONE;
@@ -15695,7 +15704,7 @@ void EmitPass::emitTypedMemoryFence(llvm::Instruction* inst)
 
     if (shouldGenerateLSC())
     {
-        auto flushOpt = m_currShader->m_Platform->hasSamplerSupport() ? LSC_FENCE_OP_EVICT : getLSCMemoryFenceOp(true, L1_Invalidate);
+        auto flushOpt = getLSCMemoryFenceOp(true, L1_Invalidate, m_currShader->m_Platform->hasSamplerSupport());
         LSC_SCOPE scope = LSC_SCOPE_GPU;
         if (!m_currShader->m_Platform->hasMultiTile() &&
             m_currShader->m_Platform->hasL3FlushOnGPUScopeInvalidate() &&
@@ -21724,7 +21733,7 @@ void EmitPass::emitSystemMemoryFence(llvm::GenIntrinsicInst* inst)
     if (fenceTGM)
     {
         // first fence TGM with GPU scope
-        auto flushOpt = m_currShader->m_Platform->hasSamplerSupport() ? LSC_FENCE_OP_EVICT : getLSCMemoryFenceOp(true, false);
+        auto flushOpt = getLSCMemoryFenceOp(true, false, m_currShader->m_Platform->hasSamplerSupport());
         m_encoder->LSC_Fence(LSC_TGM, LSC_SCOPE_GPU, flushOpt);
         m_encoder->Push();
         // then emit the regular UGM fence
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp
@@ -886,7 +886,7 @@ class EmitPass : public llvm::FunctionPass
         LSC_CACHE_OPTS cacheOpts, bool UseA32,
         ResourceDescriptor& Resource, CVariable* StoreVar, CVariable* Offset, int ImmOffset,
         uint32_t NumElts, uint32_t EltBytes, int Align);
-    LSC_FENCE_OP getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1) const;
+    LSC_FENCE_OP getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1, bool EvictL1) const;
     bool m_isDuplicate;
     CVariable* m_tmpDest = nullptr;
 
diff --git a/IGC/Compiler/CISACodeGen/PixelShaderLowering.cpp b/IGC/Compiler/CISACodeGen/PixelShaderLowering.cpp
@@ -571,6 +571,7 @@ void PixelShaderLowering::EmitMemoryFence(IRBuilder<>& builder, bool forceFlushN
         falseValue,
         trueValue,
         falseValue,
+        falseValue,
     };
 
     CallInst* memFence = GenIntrinsicInst::Create(GenISAIntrinsic::getDeclaration(m_module, GenISAIntrinsic::GenISA_memoryfence),
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/Atomics/ResolveOCLAtomics.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/Atomics/ResolveOCLAtomics.cpp
@@ -358,7 +358,7 @@ void ResolveOCLAtomics::generateLockInitilization(Function* F)
     m_builder->CreateStore(m_localLock->getInitializer(), m_localLock);
     m_builder->CreateBr(initSpinLockEndBB);
 
-    // insert call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true)
+    // insert call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false)
     //        call void @llvm.genx.GenISA.threadgroupbarrier()
     // to guarantee synchronization in accessing spin lock variable
     Value* trueValue = m_builder->getTrue();
@@ -372,6 +372,7 @@ void ResolveOCLAtomics::generateLockInitilization(Function* F)
         falseValue,
         falseValue,
         trueValue,
+        falseValue,
     };
     m_builder->SetInsertPoint(initSpinLockEndBB, initSpinLockEndBB->getFirstInsertionPt());
     Function* localMemFence = GenISAIntrinsic::getDeclaration(m_pModule, GenISAIntrinsic::GenISA_memoryfence);
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/NamedBarriers/NamedBarriersResolution.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/NamedBarriers/NamedBarriersResolution.cpp
@@ -241,6 +241,7 @@ void NamedBarriersResolution::HandleNamedBarrierSyncPVC(CallInst& NBarrierSyncCa
             falseValue, // bool flushIcache
             isGlobal,   // bool isGlobal
             falseValue, // bool invalidateL1
+            falseValue, // bool evictL1
         },
         "",
         &(NBarrierSyncCall));
diff --git a/IGC/GenISAIntrinsics/Intrinsic_definitions.py b/IGC/GenISAIntrinsics/Intrinsic_definitions.py
@@ -1410,7 +1410,8 @@
      ("bool",                          "L3_Flush_Texture_Data"),
      ("bool",                          "L3_Flush_Instructions"),
      ("bool",                          "Fence has global effect"),
-     ("bool",                          "L1 Invalidate")],
+     ("bool",                          "L1 Invalidate"),
+     ("bool",                          "L1 Evict")],
     "Convergent"]],
 ####################################################################################################
 "GenISA_mov_identity": ["",

Original file line number	Diff line number	Diff line change
`@@ -15551,14 +15551,19 @@ void EmitPass::emitThreadGroupBarrier(llvm::Instruction* inst)`
`15551`	`15551`	`}`
`15552`	`15552`	`}`
`15553`	`15553`
`15554`		`-LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1) const`
	`15554`	`+LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool InvalidateL1, bool EvictL1) const`
`15555`	`15555`	`{`
`15556`	`15556`	`LSC_FENCE_OP op = LSC_FENCE_OP_NONE;`
`15557`	`15557`	`if (InvalidateL1 \|\| (IsGlobalMemFence && m_currShader->m_Platform->getWATable().Wa_14012437816))`
`15558`	`15558`	`{`
`15559`	`15559`	`op = LSC_FENCE_OP_INVALIDATE;`
`15560`	`15560`	`}`
`15561`	`15561`
	`15562`	`+ if (EvictL1)`
	`15563`	`+ {`
	`15564`	`+ op = LSC_FENCE_OP_EVICT;`
	`15565`	`+ }`
	`15566`	`+`
`15562`	`15567`	`// For experiment on XeHP SDV`
`15563`	`15568`	`if (op == LSC_FENCE_OP_NONE && IsGlobalMemFence && IGC_IS_FLAG_ENABLED(EnableL3FlushForGlobal))`
`15564`	`15569`	`{`
`@@ -15569,7 +15574,7 @@ LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool Invalidat`
`15569`	`15574`
`15570`	`15575`	`void EmitPass::emitMemoryFence(llvm::Instruction* inst)`
`15571`	`15576`	`{`
`15572`		`- static constexpr int ExpectedNumberOfArguments = 7;`
	`15577`	`+ static constexpr int ExpectedNumberOfArguments = 8;`
`15573`	`15578`	`IGC_ASSERT(cast<CallInst>(inst)->getNumArgOperands() == ExpectedNumberOfArguments);`
`15574`	`15579`	`CodeGenContext* ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();`
`15575`	`15580`
`@@ -15583,6 +15588,9 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)`
`15583`	`15588`	`bool L3_Flush_Instructions = true;`
`15584`	`15589`	`bool Global_Mem_Fence = true;`
`15585`	`15590`	`bool L1_Invalidate = ctx->platform.hasL1ReadOnlyCache();`
	`15591`	`+ // Note: this flag is respected only for LSC case currently.`
	`15592`	`+ // TODO: add support for non-LSC and typed fence.`
	`15593`	`+ bool L1_Evict = true;`
`15586`	`15594`
`15587`	`15595`	`std::array<reference_wrapper<bool>, ExpectedNumberOfArguments> MemFenceArguments{`
`15588`	`15596`	`CommitEnable,`
`@@ -15592,6 +15600,7 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)`
`15592`	`15600`	`L3_Flush_Instructions,`
`15593`	`15601`	`Global_Mem_Fence,`
`15594`	`15602`	`L1_Invalidate,`
	`15603`	`+ L1_Evict`
`15595`	`15604`	`};`
`15596`	`15605`
`15597`	`15606`	`for (size_t i = 0; i < MemFenceArguments.size(); ++i) {`
`@@ -15639,7 +15648,7 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)`
`15639`	`15648`	`{`
`15640`	`15649`	`scope = LSC_SCOPE_TILE;`
`15641`	`15650`	`}`
`15642`		`- LSC_FENCE_OP op = getLSCMemoryFenceOp(Global_Mem_Fence, L1_Invalidate);`
	`15651`	`+ LSC_FENCE_OP op = getLSCMemoryFenceOp(Global_Mem_Fence, L1_Invalidate, L1_Evict);`
`15643`	`15652`	`if (inst->getMetadata("forceFlushNone"))`
`15644`	`15653`	`{`
`15645`	`15654`	`op = LSC_FENCE_OP_NONE;`
`@@ -15695,7 +15704,7 @@ void EmitPass::emitTypedMemoryFence(llvm::Instruction* inst)`
`15695`	`15704`
`15696`	`15705`	`if (shouldGenerateLSC())`
`15697`	`15706`	`{`
`15698`		`- auto flushOpt = m_currShader->m_Platform->hasSamplerSupport() ? LSC_FENCE_OP_EVICT : getLSCMemoryFenceOp(true, L1_Invalidate);`
	`15707`	`+ auto flushOpt = getLSCMemoryFenceOp(true, L1_Invalidate, m_currShader->m_Platform->hasSamplerSupport());`
`15699`	`15708`	`LSC_SCOPE scope = LSC_SCOPE_GPU;`
`15700`	`15709`	`if (!m_currShader->m_Platform->hasMultiTile() &&`
`15701`	`15710`	`m_currShader->m_Platform->hasL3FlushOnGPUScopeInvalidate() &&`
`@@ -21724,7 +21733,7 @@ void EmitPass::emitSystemMemoryFence(llvm::GenIntrinsicInst* inst)`
`21724`	`21733`	`if (fenceTGM)`
`21725`	`21734`	`{`
`21726`	`21735`	`// first fence TGM with GPU scope`
`21727`		`- auto flushOpt = m_currShader->m_Platform->hasSamplerSupport() ? LSC_FENCE_OP_EVICT : getLSCMemoryFenceOp(true, false);`
	`21736`	`+ auto flushOpt = getLSCMemoryFenceOp(true, false, m_currShader->m_Platform->hasSamplerSupport());`
`21728`	`21737`	`m_encoder->LSC_Fence(LSC_TGM, LSC_SCOPE_GPU, flushOpt);`
`21729`	`21738`	`m_encoder->Push();`
`21730`	`21739`	`// then emit the regular UGM fence`