Add possibility to lower llvm.fmuladd into mad from OpenCL extinst

Author: AlexeySachkov

llvm-spirv/include/LLVMSPIRVOpts.h

@@ -160,6 +160,14 @@ class TranslatorOpts {
 
   void setDebugInfoEIS(DebugInfoEIS EIS) { DebugInfoVersion = EIS; }
 
+  bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept {
+    return ReplaceLLVMFmulAddWithOpenCLMad;
+  }
+
+  void setReplaceLLVMFmulAddWithOpenCLMad(bool Value) noexcept {
+    ReplaceLLVMFmulAddWithOpenCLMad = Value;
+  }
+
 private:
   // Common translation options
   VersionNumber MaxVersion = VersionNumber::MaximumVersion;
@@ -192,6 +200,10 @@ class TranslatorOpts {
   bool AllowExtraDIExpressions = false;
 
   DebugInfoEIS DebugInfoVersion = DebugInfoEIS::OpenCL_DebugInfo_100;
+
+  // Controls whether llvm.fmuladd.* should be replaced with mad from OpenCL
+  // extended instruction set or with a simple fmul + fadd
+  bool ReplaceLLVMFmulAddWithOpenCLMad = true;
 };
 
 } // namespace SPIRV

llvm-spirv/lib/SPIRV/SPIRVWriter.cpp

@@ -2337,6 +2337,19 @@ SPIRVValue *LLVMToSPIRV::transIntrinsicInst(IntrinsicInst *II,
     // For llvm.fmuladd.* fusion is not guaranteed. If a fused multiply-add
     // is required the corresponding llvm.fma.* intrinsic function should be
     // used instead.
+    // If allowed, let's replace llvm.fmuladd.* with mad from OpenCL extended
+    // instruction set, as it has the same semantic for FULL_PROFILE OpenCL
+    // devices (implementation-defined for EMBEDDED_PROFILE).
+    if (BM->shouldReplaceLLVMFmulAddWithOpenCLMad()) {
+      std::vector<SPIRVValue *> Ops{transValue(II->getArgOperand(0), BB),
+                                    transValue(II->getArgOperand(1), BB),
+                                    transValue(II->getArgOperand(2), BB)};
+      return BM->addExtInst(transType(II->getType()),
+                            BM->getExtInstSetId(SPIRVEIS_OpenCL),
+                            OpenCLLIB::Mad, Ops, BB);
+    }
+
+    // Otherwise, just break llvm.fmuladd.* into a pair of fmul + fadd
     SPIRVType *Ty = transType(II->getType());
     SPIRVValue *Mul =
         BM->addBinaryInst(OpFMul, Ty, transValue(II->getArgOperand(0), BB),

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVModule.h

@@ -489,6 +489,10 @@ class SPIRVModule {
     return TranslationOpts.allowExtraDIExpressions();
   }
 
+  bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept {
+    return TranslationOpts.shouldReplaceLLVMFmulAddWithOpenCLMad();
+  }
+
   SPIRVExtInstSetKind getDebugInfoEIS() const {
     switch (TranslationOpts.getDebugInfoEIS()) {
     case DebugInfoEIS::SPIRV_Debug:

llvm-spirv/test/llvm-intrinsics/fmuladd.ll

@@ -1,12 +1,25 @@
 ; RUN: llvm-as %s -o %t.bc
-; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s
-; RUN: llvm-spirv %t.bc -o %t.spv
-; RUN: spirv-val %t.spv
-
-; CHECK-NOT: llvm.fmuladd
-
-; CHECK: TypeFloat [[f32:[0-9]+]] 32
-; CHECK: TypeFloat [[f64:[0-9]+]] 64
+; RUN: llvm-spirv %t.bc -o %default.spv
+; RUN: llvm-spirv %t.bc --spirv-replace-fmuladd-with-ocl-mad=true -o %replace.spv
+; RUN: llvm-spirv %t.bc --spirv-replace-fmuladd-with-ocl-mad=false -o %break.spv
+; RUN: spirv-val %replace.spv
+; RUN: spirv-val %break.spv
+; RUN: llvm-spirv %default.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,REPLACE
+; RUN: llvm-spirv %replace.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,REPLACE
+; RUN: llvm-spirv %break.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,BREAK
+
+; COMMON-NOT: llvm.fmuladd
+
+; COMMON: TypeFloat [[f32:[0-9]+]] 32
+; COMMON: TypeFloat [[f64:[0-9]+]] 64
+;
+; REPLACE: ExtInst [[f32]] {{[0-9]+}} {{[0-9]+}} mad
+; REPLACE: ExtInst [[f64]] {{[0-9]+}} {{[0-9]+}} mad
+;
+; BREAK: FMul [[f32]] [[mul32:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
+; BREAK-NEXT: FAdd [[f32]] {{[0-9]+}} [[mul32]] {{[0-9]+}}
+; BREAK: FMul [[f64]] [[mul64:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
+; BREAK-NEXT: FAdd [[f64]] {{[0-9]+}} [[mul64]] {{[0-9]+}}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
 target triple = "spir64"
@@ -15,11 +28,7 @@ target triple = "spir64"
 define spir_func void @foo(float %a, float %b, float %c, double %x, double %y, double %z) #0 {
 entry:
   %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
-; CHECK: FMul [[f32]] [[mul32:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
-; CHECK-NEXT: FAdd [[f32]] {{[0-9]+}} [[mul32]] {{[0-9]+}}
   %1 = call double @llvm.fmuladd.f64(double %x, double %y, double %z)
-; CHECK: FMul [[f64]] [[mul64:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
-; CHECK-NEXT: FAdd [[f64]] {{[0-9]+}} [[mul64]] {{[0-9]+}}
 ret void
 }
 

llvm-spirv/tools/llvm-spirv/llvm-spirv.cpp

@@ -202,6 +202,12 @@ static cl::opt<SPIRV::DebugInfoEIS> DebugEIS(
                    "extended instruction set. This version of SPIR-V debug "
                    "info format is compatible with the SPIRV-Tools")));
 
+static cl::opt<bool> SPIRVReplaceLLVMFmulAddWithOpenCLMad(
+    "spirv-replace-fmuladd-with-ocl-mad",
+    cl::desc("Allow replacement of llvm.fmuladd.* intrinsic with OpenCL mad "
+             "instruction from OpenCL extended instruction set"),
+    cl::init(true));
+
 static std::string removeExt(const std::string &FileName) {
   size_t Pos = FileName.find_last_of(".");
   if (Pos != std::string::npos)
@@ -587,6 +593,16 @@ int main(int Ac, char **Av) {
     }
   }
 
+  if (SPIRVReplaceLLVMFmulAddWithOpenCLMad.getNumOccurrences() != 0) {
+    if (IsReverse) {
+      errs() << "Note: --spirv-replace-fmuladd-with-ocl-mad option ignored as "
+                "it only affects translation from LLVM IR to SPIR-V";
+    } else {
+      Opts.setReplaceLLVMFmulAddWithOpenCLMad(
+          SPIRVReplaceLLVMFmulAddWithOpenCLMad);
+    }
+  }
+
   if (SPIRVAllowExtraDIExpressions.getNumOccurrences() != 0) {
     Opts.setAllowExtraDIExpressionsEnabled(SPIRVAllowExtraDIExpressions);
   }