Add support for ".uniform" modifier in ifcall instruction

pratikashar · igcbot · commit 5136a857d678 · 2023-08-25T06:30:52.000+02:00
Add support for ".uniform" modifier in ifcall instruction.

When IGC marks an ifcall as uniform, VISA skips running special EU
fusion call WA when -fusedCallWA = 2.
diff --git a/IGC/Compiler/CISACodeGen/CISABuilder.cpp b/IGC/Compiler/CISACodeGen/CISABuilder.cpp
@@ -416,7 +416,8 @@ namespace IGC
         VISA_EMask_Ctrl emask = m_encoderState.m_noMask ? vISA_EMASK_M1_NM : vISA_EMASK_M1;
         VISA_Exec_Size execSize = visaExecSize(m_program->m_dispatchSize);
         VISA_VectorOpnd* funcAddrOpnd = GetSourceOperandNoModifier(funcPtr);
-        V(vKernel->AppendVISACFIndirectFuncCallInst(predOpnd, emask, execSize, funcAddrOpnd, argSize, retSize));
+        V(vKernel->AppendVISACFIndirectFuncCallInst(
+            predOpnd, emask, execSize, false, funcAddrOpnd, argSize, retSize));
     }
 
     void CEncoder::SubroutineRet(CVariable* flag, llvm::Function* F)
diff --git a/IGC/VectorCompiler/lib/GenXCodeGen/GenXCisaBuilder.cpp b/IGC/VectorCompiler/lib/GenXCodeGen/GenXCisaBuilder.cpp
@@ -867,7 +867,7 @@ class GenXKernelBuilder {
                                                unsigned char ReturnSize) {
     updateSIMDSize(ExecMask, ExecSize);
     CISA_CALL(Kernel->AppendVISACFIndirectFuncCallInst(
-        Pred, ExecMask, ExecSize, FuncAddr, ArgSize, ReturnSize));
+        Pred, ExecMask, ExecSize, false, FuncAddr, ArgSize, ReturnSize));
   }
 
   inline void appendVISACFRetInst(VISA_PredOpnd *Pred, VISA_EMask_Ctrl ExecMask,
diff --git a/documentation/visa/instructions/IFCALL.md b/documentation/visa/instructions/IFCALL.md
@@ -87,7 +87,7 @@ SPDX-License-Identifier: MIT
 
 
 
-    [(<P>)] IFCALL (<exec_size>) <func_id> <arg_size> <return_size>
+    [(<P>)] IFCALL.[uniform] (<exec_size>) <func_id> <arg_size> <return_size>
 ```
 ## Notes
 
@@ -96,6 +96,8 @@ SPDX-License-Identifier: MIT
 
 
 ```
+    Optional .uniform modifier is set if all threads invoke same callee.
+
     If <exec_size> is one: Execution jumps to the function if the predicate is true. The call mask will be initialized to all ones at function entry. Scalar calls must be marked with {NoMask}.
 
     If <exec_size> is greater than one: The call is executed if any of the active channels are predicated. At function entry, the call mask will be initialized to the set of active channels that are predicated.
diff --git a/visa/BuildCISAIR.h b/visa/BuildCISAIR.h
@@ -487,8 +487,9 @@ class CISA_IR_Builder : public VISABuilder {
 
   bool CISA_create_ifcall_instruction(VISA_opnd *pred_opnd,
                                       VISA_EMask_Ctrl emask, unsigned exec_size,
-                                      VISA_opnd *funcAddr, unsigned arg_size,
-                                      unsigned return_size, int lineNum);
+                                      bool isUniform, VISA_opnd *funcAddr,
+                                      unsigned arg_size, unsigned return_size,
+                                      int lineNum);
 
   bool CISA_create_faddr_instruction(const char *sym_name, VISA_opnd *dst,
                                      int lineNum);
diff --git a/visa/BuildCISAIRImpl.cpp b/visa/BuildCISAIRImpl.cpp
@@ -1177,7 +1177,8 @@ void CISA_IR_Builder::LinkTimeOptimization(
             auto orig_fcallinfo = callee->fg.builder->getFcallInfo(fret);
             if (orig_fcallinfo) {
               builder->addFcallInfo(inst, orig_fcallinfo->getArgSize(),
-                  orig_fcallinfo->getRetSize());
+                                    orig_fcallinfo->getRetSize(),
+                                    orig_fcallinfo->isUniform());
             }
           }
 
@@ -1264,8 +1265,9 @@ void CISA_IR_Builder::LinkTimeOptimization(
           if (inst->opcode() == G4_pseudo_fcall) {
             auto orig_fcallinfo = callee->fg.builder->getFcallInfo(fret);
             if (orig_fcallinfo) {
-              caller->fg.builder->addFcallInfo(inst, orig_fcallinfo->getArgSize(),
-                  orig_fcallinfo->getRetSize());
+              caller->fg.builder->addFcallInfo(
+                  inst, orig_fcallinfo->getArgSize(),
+                  orig_fcallinfo->getRetSize(), orig_fcallinfo->isUniform());
             }
           }
           for (int i = 0, numSrc = inst->getNumSrc(); i < numSrc; ++i) {
@@ -3593,12 +3595,13 @@ bool CISA_IR_Builder::CISA_create_fcall_instruction(
 
 bool CISA_IR_Builder::CISA_create_ifcall_instruction(
     VISA_opnd *pred_opnd, VISA_EMask_Ctrl emask, unsigned exec_size,
-    VISA_opnd *funcAddr, unsigned arg_size, unsigned return_size,
+    bool isUniform, VISA_opnd *funcAddr, unsigned arg_size,
+    unsigned return_size,
     int lineNum) // last index
 {
   VISA_Exec_Size executionSize = Get_VISA_Exec_Size_From_Raw_Size(exec_size);
   VISA_CALL_TO_BOOL(AppendVISACFIndirectFuncCallInst,
-                    (VISA_PredOpnd *)pred_opnd, emask, executionSize,
+                    (VISA_PredOpnd *)pred_opnd, emask, executionSize, isUniform,
                     (VISA_VectorOpnd *)funcAddr, (uint8_t)arg_size,
                     (uint8_t)return_size);
   return true;
diff --git a/visa/BuildIR.h b/visa/BuildIR.h
@@ -619,8 +619,8 @@ class IR_Builder {
 
   std::optional<G4_FCALL> getFcallInfo(const G4_INST *inst) const;
   void addFcallInfo(const G4_INST *FcallInst, uint16_t ArgSize,
-                    uint16_t RetSize) {
-    m_fcallInfo.emplace(FcallInst, G4_FCALL(ArgSize, RetSize));
+                    uint16_t RetSize, bool isUniform) {
+    m_fcallInfo.emplace(FcallInst, G4_FCALL(ArgSize, RetSize, isUniform));
   }
 
   // If this is true (detected in TranslateInterface.cpp), we need a sampler
@@ -1573,8 +1573,9 @@ class IR_Builder {
                                uint8_t argSize, uint8_t returnSize);
 
   int translateVISACFIFCallInst(VISA_Exec_Size execsize, VISA_EMask_Ctrl emask,
-                                G4_Predicate *predOpnd, G4_Operand *funcAddr,
-                                uint8_t argSize, uint8_t returnSize);
+                                G4_Predicate *predOpnd, bool isUniform,
+                                G4_Operand *funcAddr, uint8_t argSize,
+                                uint8_t returnSize);
 
   int translateVISACFSymbolInst(const std::string &symbolName,
                                 G4_DstRegRegion *dst);
diff --git a/visa/ByteCodeReaderNG.cpp b/visa/ByteCodeReaderNG.cpp
@@ -1194,12 +1194,13 @@ static void readInstructionControlFlow(unsigned &bytePos, const char *buf,
                               ? readPredicateOperandNG(bytePos, buf, container)
                               : nullptr;
 
+    bool isUniform = false;
     VISA_VectorOpnd *funcAddr =
         readVectorOperandNG(bytePos, buf, container, false);
     uint8_t argSize = readPrimitiveOperandNG<uint8_t>(bytePos, buf);
     uint8_t retSize = readPrimitiveOperandNG<uint8_t>(bytePos, buf);
-    kernelBuilder->AppendVISACFIndirectFuncCallInst(pred, emask, esize,
-                                                    funcAddr, argSize, retSize);
+    kernelBuilder->AppendVISACFIndirectFuncCallInst(
+        pred, emask, esize, isUniform, funcAddr, argSize, retSize);
     return;
   }
   case ISA_FADDR: {
diff --git a/visa/CISA.l b/visa/CISA.l
@@ -1228,6 +1228,11 @@ arg    {return LSC_AM_ARG;}
         return IDENT;
     }
 
+".uniform" {
+        TRACE("** UNIFORM");
+        CISAlval.string = strdup(yytext);
+        return UNIFORM;
+    }
 
 
 
diff --git a/visa/CISA.y b/visa/CISA.y
@@ -394,6 +394,7 @@ std::vector<attr_gen_struct*> AttrOptVar;
 %token NEWLINE
 
 
+%token                     UNIFORM
 %token <string>            RTWRITE_OPTION
 %token <media_mode>        MEDIA_MODE
 %token <oword_mod>         OWORD_MODIFIER
@@ -2259,9 +2260,16 @@ BranchInstruction: Predicate BRANCH_OP ExecSize IdentOrStringLit
     | Predicate IFCALL ExecSize VecSrcOperand_G_I_IMM DEC_LIT DEC_LIT
     {
         pBuilder->CISA_create_ifcall_instruction(
-        $1, $3.emask, $3.exec_size,
+        $1, $3.emask, $3.exec_size, false,
         $4.cisa_gen_opnd, (unsigned)$5, (unsigned)$6, CISAlineno);
     }
+    // 1          2       3       4           5                  6       7
+    | Predicate IFCALL UNIFORM ExecSize VecSrcOperand_G_I_IMM DEC_LIT DEC_LIT
+    {
+        pBuilder->CISA_create_ifcall_instruction(
+        $1, $4.emask, $4.exec_size, true,
+        $5.cisa_gen_opnd, (unsigned)$6, (unsigned)$7, CISAlineno);
+    }
     // 1      2                3
     | FADDR  IdentOrStringLit VecDstOperand_G_I
     {
diff --git a/visa/G4_IR.hpp b/visa/G4_IR.hpp
@@ -87,14 +87,16 @@ class BinInst;
 class G4_FCALL {
   uint16_t argSize;
   uint16_t retSize;
+  bool uniform = false;
 
 public:
   G4_FCALL() = delete;
-  G4_FCALL(uint16_t argVarSz, uint16_t retVarSz)
-      : argSize(argVarSz), retSize(retVarSz) {}
+  G4_FCALL(uint16_t argVarSz, uint16_t retVarSz, bool isUniform)
+      : argSize(argVarSz), retSize(retVarSz), uniform(isUniform) {}
 
   uint16_t getArgSize() const { return argSize; }
   uint16_t getRetSize() const { return retSize; }
+  bool isUniform() const { return uniform; }
 };
 
 // Forward references for classes used by G4_INST.
diff --git a/visa/IsaDescription.cpp b/visa/IsaDescription.cpp
@@ -100,7 +100,7 @@ struct ISA_Inst_Info ISA_Inst_Table[ISA_OPCODE_ENUM_SIZE] = {
     {ISA_SCATTER4_TYPED, ISA_Inst_Data_Port, "scatter4_typed", 8, 0},
     {ISA_VA_SKL_PLUS, ISA_Inst_Sampler, "va_skl_plus", 0, 0},
     {ISA_SVM, ISA_Inst_SVM, "svm", 0, 0},
-    {ISA_IFCALL, ISA_Inst_Flow, "ifcall", 3, 0},
+    {ISA_IFCALL, ISA_Inst_Flow, "ifcall", 4, 0},
     {ISA_FADDR, ISA_Inst_Flow, "faddr", 1, 1},
     {ISA_FILE, ISA_Inst_Misc, "file", 1, 0},
     {ISA_LOC, ISA_Inst_Misc, "loc", 1, 0},
@@ -1558,11 +1558,12 @@ VISA_INST_Desc CISA_INST_table[ISA_NUM_OPCODE] = {
         ISA_IFCALL,
         ISA_Inst_Flow,
         "ifcall",
-        5,
+        6,
         0,
         {
             {OPND_EXECSIZE, ISA_TYPE_UB, 0},
             {OPND_PRED, ISA_TYPE_UW, 0},
+            {OPND_KIND, ISA_TYPE_UB, 0}, /// isUniform
             {OPND_KIND, ISA_TYPE_UD, 0}, /// function_addr
             {OPND_KIND, ISA_TYPE_UB, 0}, /// arg_size
             {OPND_KIND, ISA_TYPE_UB, 0}, /// return_size
diff --git a/visa/IsaDisassembly.cpp b/visa/IsaDisassembly.cpp
@@ -980,13 +980,14 @@ printInstructionControlFlow(const print_format_provider_t *header,
       sstr << uniqueSuffixStr;
   } else {
     sstr << printPredicate(inst->opcode, inst->pred)
-         << ISA_Inst_Table[opcode].str << " "
-         << printExecutionSize(inst->opcode, inst->execsize);
+         << ISA_Inst_Table[opcode].str;
 
     switch (opcode) {
     case ISA_JMP:
     case ISA_GOTO:
     case ISA_FCALL: {
+      sstr << " " << printExecutionSize(inst->opcode, inst->execsize);
+
       /// label / function id to jump / call to.
       label_id = getPrimitiveOperand<uint32_t>(inst, i++);
 
@@ -1013,6 +1014,11 @@ printInstructionControlFlow(const print_format_provider_t *header,
       break;
     }
     case ISA_IFCALL: {
+      /// isUniform
+      auto isUniform = getPrimitiveOperand<unsigned>(inst, i++);
+      if (isUniform)
+        sstr << ".uniform";
+      sstr << " " << printExecutionSize(inst->opcode, inst->execsize);
       sstr << printOperand(header, inst, i++, opt);
       /// arg size
       sstr << " " << getPrimitiveOperand<unsigned>(inst, i++);
@@ -1021,6 +1027,7 @@ printInstructionControlFlow(const print_format_provider_t *header,
       break;
     }
     case ISA_FADDR: {
+      sstr << " " << printExecutionSize(inst->opcode, inst->execsize);
       /// symbol name in string
       const char *sym =
           header->getString(getPrimitiveOperand<uint32_t>(inst, i++));
@@ -1034,6 +1041,7 @@ printInstructionControlFlow(const print_format_provider_t *header,
       break;
     }
     case ISA_SWITCHJMP: {
+      sstr << " " << printExecutionSize(inst->opcode, inst->execsize);
       /// skip num_labels
       i++;
       /// index
@@ -1053,6 +1061,7 @@ printInstructionControlFlow(const print_format_provider_t *header,
       break;
     }
     default:
+      sstr << " " << printExecutionSize(inst->opcode, inst->execsize);
       break; // Prevent gcc warning
     }
   }
diff --git a/visa/SWWA.cpp b/visa/SWWA.cpp
@@ -2243,8 +2243,9 @@ void Optimizer::applyFusedCallWA() {
     }
   };
 
+  unsigned int fusedEUCallWA = builder.getuint32Option(vISA_fusedCallWA);
   // Only process call wa (fusedCallWA = 1) or indirect call is non-uniform
-  if (!((builder.getuint32Option(vISA_fusedCallWA) == 1) ||
+  if (!((fusedEUCallWA == 1) ||
         !builder.getOption(vISA_fusedCallUniform))) {
     return;
   }
@@ -2263,6 +2264,13 @@ void Optimizer::applyFusedCallWA() {
       continue;
     }
 
+    if (fusedEUCallWA == 2) {
+      auto callInfo = builder.getFcallInfo(callI);
+      vISA_ASSERT(callInfo, "call info absent for ifcall");
+      if (callInfo->isUniform())
+        continue;
+    }
+
     // Assume fcall always have a single/fall-thru succ
     if (BI == BE || BB->Succs.size() != 1 || BB->Succs.back() != (*BI)) {
       // Skip! (Could this happen ?)
@@ -2410,7 +2418,8 @@ void Optimizer::applyFusedCallWA() {
       auto orig_fcallinfo = builder.getFcallInfo(callI);
       if (orig_fcallinfo) {
         builder.addFcallInfo(nCallI, orig_fcallinfo->getArgSize(),
-                             orig_fcallinfo->getRetSize());
+                             orig_fcallinfo->getRetSize(),
+                             orig_fcallinfo->isUniform());
       }
       // Might need to update subroutine table
       updateSubroutineTableIfNeeded(origNextBB, bigB0, bigB1, smallB0, smallB1,
@@ -2635,7 +2644,8 @@ void Optimizer::applyFusedCallWA() {
     auto orig_fcallinfo = builder.getFcallInfo(callI);
     if (orig_fcallinfo) {
       builder.addFcallInfo(nCallI, orig_fcallinfo->getArgSize(),
-                           orig_fcallinfo->getRetSize());
+                           orig_fcallinfo->getRetSize(),
+                           orig_fcallinfo->isUniform());
     }
     // Might need to update subroutine table
     updateSubroutineTableIfNeeded(origNextBB, bigB0, bigB1, smallB0, smallB1,
diff --git a/visa/VISAKernel.h b/visa/VISAKernel.h
@@ -508,9 +508,11 @@ class VISAKernelImpl : public VISAFunction {
                                std::string funcName, unsigned char argSize,
                                unsigned char returnSize) override;
 
-  VISA_BUILDER_API int AppendVISACFIndirectFuncCallInst(
-      VISA_PredOpnd *pred, VISA_EMask_Ctrl emask, VISA_Exec_Size executionSize,
-      VISA_VectorOpnd *funcAddr, uint8_t argSize, uint8_t returnSize) override;
+  VISA_BUILDER_API int
+  AppendVISACFIndirectFuncCallInst(VISA_PredOpnd *pred, VISA_EMask_Ctrl emask,
+                                   VISA_Exec_Size executionSize, bool isUniform,
+                                   VISA_VectorOpnd *funcAddr, uint8_t argSize,
+                                   uint8_t returnSize) override;
 
   VISA_BUILDER_API int AppendVISACFSymbolInst(std::string symbolName,
                                               VISA_VectorOpnd *dst) override;
diff --git a/visa/VISAKernelImpl.cpp b/visa/VISAKernelImpl.cpp
@@ -3347,24 +3347,30 @@ int VISAKernelImpl::AppendVISACFFunctionCallInst(
 
 int VISAKernelImpl::AppendVISACFIndirectFuncCallInst(
     VISA_PredOpnd *pred, VISA_EMask_Ctrl emask, VISA_Exec_Size executionSize,
-    VISA_VectorOpnd *funcAddr, uint8_t argSize, uint8_t returnSize) {
+    bool isUniform, VISA_VectorOpnd *funcAddr, uint8_t argSize,
+    uint8_t returnSize) {
   TIME_SCOPE(VISA_BUILDER_APPEND_INST);
 
   AppendVISAInstCommon();
 
   int status = VISA_SUCCESS;
   if (IS_GEN_BOTH_PATH) {
     G4_Predicate *g4Pred = pred ? pred->g4opnd->asPredicate() : nullptr;
-    status = m_builder->translateVISACFIFCallInst(
-        executionSize, emask, g4Pred, funcAddr->g4opnd, argSize, returnSize);
+    status = m_builder->translateVISACFIFCallInst(executionSize, emask, g4Pred,
+                                                  isUniform, funcAddr->g4opnd,
+                                                  argSize, returnSize);
   }
   if (IS_VISA_BOTH_PATH) {
     VISA_INST_Desc *inst_desc = &CISA_INST_table[ISA_IFCALL];
-    VISA_opnd *opnd[3]; // should be more then enough
+    VISA_opnd *opnd[4]; // should be more then enough
     int num_pred_desc_operands = 2;
     GET_NUM_PRED_DESC_OPNDS(num_pred_desc_operands, inst_desc);
     int num_operands = 0;
 
+    ADD_OPND(num_operands, opnd,
+             CreateOtherOpndHelper(num_pred_desc_operands, num_operands,
+                                   inst_desc, isUniform));
+
     opnd[num_operands] = funcAddr;
     ++num_operands;
 
diff --git a/visa/VisaToG4/TranslateBranches.cpp b/visa/VisaToG4/TranslateBranches.cpp
@@ -123,13 +123,14 @@ int IR_Builder::translateVISACFFCallInst(VISA_Exec_Size execsize,
   auto fcall = createInst(predOpnd, G4_pseudo_fcall, nullptr, g4::NOSAT, exsize,
                           nullptr, callLabel, nullptr, InstOpt_NoOpt, true);
 
-  addFcallInfo(fcall, argSize, returnSize);
+  addFcallInfo(fcall, argSize, returnSize, false);
   return VISA_SUCCESS;
 }
 
 int IR_Builder::translateVISACFIFCallInst(VISA_Exec_Size execsize,
                                           VISA_EMask_Ctrl emask,
                                           G4_Predicate *predOpnd,
+                                          bool isUniform,
                                           G4_Operand *funcAddr, uint8_t argSize,
                                           uint8_t returnSize) {
   TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);
@@ -160,7 +161,7 @@ int IR_Builder::translateVISACFIFCallInst(VISA_Exec_Size execsize,
   auto fcall = createInst(predOpnd, G4_pseudo_fcall, nullptr, g4::NOSAT, exsize,
                           nullptr, src0, nullptr, InstOpt_NoOpt, true);
 
-  addFcallInfo(fcall, argSize, returnSize);
+  addFcallInfo(fcall, argSize, returnSize, isUniform);
 
   return VISA_SUCCESS;
 }
diff --git a/visa/include/VISABuilderAPIDefinition.h b/visa/include/VISABuilderAPIDefinition.h
@@ -643,7 +643,7 @@ class VISAKernel {
   /// [0, sizeof(Arg)] returnSize must be [0, sizeof(RetVal)]
   VISA_BUILDER_API virtual int AppendVISACFIndirectFuncCallInst(
       VISA_PredOpnd *pred, VISA_EMask_Ctrl emask, VISA_Exec_Size executionSize,
-      VISA_VectorOpnd *funcAddr, unsigned char argSize,
+      bool isUniform, VISA_VectorOpnd *funcAddr, unsigned char argSize,
       unsigned char returnSize) = 0;
 
   /// AppendVISACFFuncAddrInst -- stores the address of a symbol <symbolName>

Original file line number	Diff line number	Diff line change
`@@ -416,7 +416,8 @@ namespace IGC`
`416`	`416`	`VISA_EMask_Ctrl emask = m_encoderState.m_noMask ? vISA_EMASK_M1_NM : vISA_EMASK_M1;`
`417`	`417`	`VISA_Exec_Size execSize = visaExecSize(m_program->m_dispatchSize);`
`418`	`418`	`VISA_VectorOpnd* funcAddrOpnd = GetSourceOperandNoModifier(funcPtr);`
`419`		`- V(vKernel->AppendVISACFIndirectFuncCallInst(predOpnd, emask, execSize, funcAddrOpnd, argSize, retSize));`
	`419`	`+ V(vKernel->AppendVISACFIndirectFuncCallInst(`
	`420`	`+ predOpnd, emask, execSize, false, funcAddrOpnd, argSize, retSize));`
`420`	`421`	`}`
`421`	`422`
`422`	`423`	`void CEncoder::SubroutineRet(CVariable* flag, llvm::Function* F)`
Original file line number	Diff line number	Diff line change
`@@ -867,7 +867,7 @@ class GenXKernelBuilder {`
`867`	`867`	`unsigned char ReturnSize) {`
`868`	`868`	`updateSIMDSize(ExecMask, ExecSize);`
`869`	`869`	`CISA_CALL(Kernel->AppendVISACFIndirectFuncCallInst(`
`870`		`- Pred, ExecMask, ExecSize, FuncAddr, ArgSize, ReturnSize));`
	`870`	`+ Pred, ExecMask, ExecSize, false, FuncAddr, ArgSize, ReturnSize));`
`871`	`871`	`}`
`872`	`872`
`873`	`873`	`inline void appendVISACFRetInst(VISA_PredOpnd *Pred, VISA_EMask_Ctrl ExecMask,`
Original file line number	Diff line number	Diff line change
`@@ -1228,6 +1228,11 @@ arg {return LSC_AM_ARG;}`
`1228`	`1228`	`return IDENT;`
`1229`	`1229`	`}`
`1230`	`1230`
	`1231`	`+".uniform" {`
	`1232`	`+ TRACE("** UNIFORM");`
	`1233`	`+ CISAlval.string = strdup(yytext);`
	`1234`	`+ return UNIFORM;`
	`1235`	`+ }`
`1231`	`1236`
`1232`	`1237`
`1233`	`1238`