[tmva][sofie] Disentangle PyMVA and SOFIE

cmake/modules/SearchInstalledSoftware.cmake

@@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva))
 elseif(tpython OR tmva-pymva)
   list(APPEND python_components Development)
 endif()
-if(tmva-pymva)
+if(tmva-pymva OR tmva-sofie)
   list(APPEND python_components NumPy)
 endif()
 find_package(Python3 3.9 COMPONENTS ${python_components})
@@ -1796,7 +1796,7 @@ if(tmva)
       endif()
     endif()
   endif()
-  if(tmva-pymva)
+  if(tmva-pymva OR tmva-sofie)
     if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND))
       message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required"
                           " (python executable: ${Python3_EXECUTABLE})")

tmva/pymva/CMakeLists.txt

@@ -17,17 +17,13 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     TMVA/MethodPyKeras.h
     TMVA/MethodPyRandomForest.h
     TMVA/MethodPyTorch.h
-    TMVA/RModelParser_Keras.h
-    TMVA/RModelParser_PyTorch.h
     TMVA/PyMethodBase.h
   SOURCES
     src/MethodPyAdaBoost.cxx
     src/MethodPyGTB.cxx
     src/MethodPyKeras.cxx
     src/MethodPyRandomForest.cxx
     src/MethodPyTorch.cxx
-    src/RModelParser_Keras.cxx
-    src/RModelParser_PyTorch.cxx
     src/PyMethodBase.cxx
   LIBRARIES
     Python3::NumPy
@@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     Thread
     RIO
     TMVA
-    ROOTTMVASofie
 )
 
 ROOT_ADD_TEST_SUBDIRECTORY(test)

tmva/pymva/inc/LinkDef.h

@@ -15,8 +15,4 @@
 #pragma link C++ class TMVA::MethodPyGTB+;
 #pragma link C++ class TMVA::MethodPyKeras+;
 #pragma link C++ class TMVA::MethodPyTorch+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
 #endif

tmva/pymva/test/CMakeLists.txt

@@ -11,7 +11,7 @@
 
 project(pymva-tests)
 
-set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie)
+set(Libraries Core MathCore TMVA PyMVA)
 
 # Look for needed python modules
 ROOT_FIND_PYTHON_MODULE(torch)
@@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND)
 
 # Enable tests based on available python modules
 if(ROOT_TORCH_FOUND)
-   configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
-   configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
-   configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
-   configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
    # Test PyTorch: Binary classification
 
    if (ROOT_SKLEARN_FOUND)
@@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND)
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends})
 
-   # Test RModelParser_PyTorch
-
-   if(BLAS_FOUND)
-     ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
-           LIBRARIES
-           ROOTTMVASofie
-           TMVA
-           Python3::NumPy
-           Python3::Python
-           BLAS::BLAS
-           INCLUDE_DIRS
-           SYSTEM
-           ${CMAKE_CURRENT_BINARY_DIR}
-        )
-   endif()
-
 endif(ROOT_TORCH_FOUND)
 
 if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND))
-   configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
-   configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
 
    if (ROOT_TORCH_FOUND)
       set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification)
@@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR
    ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends})
-
-   if(BLAS_FOUND)
-      ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
-         LIBRARIES
-         ROOTTMVASofie
-         PyMVA
-         Python3::NumPy
-         Python3::Python
-         BLAS::BLAS
-         INCLUDE_DIRS
-         SYSTEM
-         ${CMAKE_CURRENT_BINARY_DIR}
-     )
-  endif()
-
 endif()

tmva/sofie/test/CMakeLists.txt

@@ -107,6 +107,8 @@ endif()
 
 # Look for needed Python modules
 ROOT_FIND_PYTHON_MODULE(torch)
+ROOT_FIND_PYTHON_MODULE(keras)
+
 if (ROOT_TORCH_FOUND)
   configure_file(Conv1dModelGenerator.py  Conv1dModelGenerator.py COPYONLY)
   configure_file(Conv2dModelGenerator.py  Conv2dModelGenerator.py COPYONLY)
@@ -127,6 +129,56 @@ if (ROOT_TORCH_FOUND)
   endif()
 endif()
 
+# Any reatures that link against libpython are disabled if built with tpython=OFF
+if (tpython AND ROOT_TORCH_FOUND AND BLAS_FOUND)
+  configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
+  configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
+  configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
+  configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
+
+  # Test RModelParser_PyTorch
+  ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
+        LIBRARIES
+        ROOTTMVASofie
+        TMVA
+        Python3::NumPy
+        Python3::Python
+        BLAS::BLAS
+        INCLUDE_DIRS
+        SYSTEM
+        ${CMAKE_CURRENT_BINARY_DIR}
+  )
+endif()
+
+
+# Any reatures that link against libpython are disabled if built with tpython=OFF
+if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND)
+
+  set(unsupported_keras_version "3.10.0")
+
+  # TODO: make sure we also support the newest Keras
+  if (NOT DEFINED ROOT_KERAS_VERSION)
+    message(WARNING "Keras found, but version unknown — cannot verify compatibility.")
+  elseif (ROOT_KERAS_VERSION VERSION_LESS ${unsupported_keras_version})
+    configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
+    configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
+
+    ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
+       LIBRARIES
+       ROOTTMVASofie
+       Python3::NumPy
+       Python3::Python
+       BLAS::BLAS
+       INCLUDE_DIRS
+       SYSTEM
+       ${CMAKE_CURRENT_BINARY_DIR}
+    )
+  else()
+    message(WARNING "Keras version ${ROOT_KERAS_VERSION} is too new for the SOFIE Keras parser (only supports < ${unsupported_keras_version})")
+  endif()
+endif()
+
+
 ROOT_EXECUTABLE(emitGNN GNN/EmitGNN.cxx LIBRARIES ROOTTMVASofie)
 ROOT_ADD_TEST(tmva-sofie-EmitGNN COMMAND emitGNN)
 

tmva/pymva/test/TestRModelParserKeras.C → tmva/sofie/test/TestRModelParserKeras.C

@@ -7,10 +7,22 @@
 
 #include "TSystem.h"
 #include "TMVA/RSofieReader.hxx"
-#include "TMVA/PyMethodBase.h"
 
 constexpr float DEFAULT_TOLERANCE = 1e-6f;
 
+namespace {
+
+// Utility functions (taken from PyMethodBase in PyMVA)
+
+const char *PyStringAsString(PyObject *string)
+{
+   PyObject *encodedString = PyUnicode_AsUTF8String(string);
+   const char *cstring = PyBytes_AsString(encodedString);
+   return cstring;
+}
+
+} // namespace
+
 void GenerateModels() {
 
    FILE* fKerasModels = fopen("generateKerasModels.py", "r");
@@ -529,7 +541,7 @@ TEST(RModel, CUSTOM_OP)
     // get input name for custom (it is output of one before last)
     PyRun_String("outputName = model.get_layer(index=len(model.layers)-2).output.name",Py_single_input,fGlobalNS,fLocalNS);
     PyObject *pOutputName = PyDict_GetItemString(fLocalNS, "outputName");
-    std::string outputName = TMVA::PyMethodBase::PyStringAsString(pOutputName);
+    std::string outputName = PyStringAsString(pOutputName);
     TMVA::Experimental:: RSofieReader r;
     r.AddCustomOperator(/*OpName*/ "Scale_by_2",
                         /*input tensor names where to insert custom op */std::string("{\"" + outputName + "\"}"),
@@ -550,4 +562,4 @@ TEST(RModel, CUSTOM_OP)
     for (size_t i = 0; i < outputCustomOp.size(); ++i) {
         EXPECT_LE(std::abs(outputCustomOp[i] - pOutputCustomOp[i]), TOLERANCE);
     }
-}
+}

tmva/pymva/test/generateKerasModels.py → tmva/sofie/test/generateKerasModels.py

@@ -1,10 +1,10 @@
 import os
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 
 import numpy as np
 import tensorflow as tf
 from tensorflow.keras.models import Model,Sequential
 from tensorflow.keras.layers import Input,Dense,Activation,ReLU,LeakyReLU,BatchNormalization,Conv2D,Reshape,Concatenate,Add,Subtract,Multiply
 from tensorflow.keras.optimizers import SGD
@@ -28,7 +28,7 @@
 
 def generateSequentialModel():
     model=Sequential()
-    model.add(Dense(8,batch_size=4))
+    model.add(Dense(8))
     model.add(ReLU())
     model.add(Dense(6))
     model.add(Activation('sigmoid'))
@@ -43,7 +43,7 @@
 
 def generateBatchNormModel():
     model=Sequential()
-    model.add(Dense(4,batch_size=2))
+    model.add(Dense(4))
     model.add(BatchNormalization())
     model.add(Dense(2))
 
@@ -180,4 +180,4 @@
 generateBinaryOpModel()
 generateActivationModel()
 generateSwishModel()
-generateCustomModel()
+generateCustomModel()

tmva/sofie_parsers/CMakeLists.txt

@@ -21,6 +21,8 @@ set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS -Wno-unused-p
 ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofieParser
   HEADERS
    TMVA/RModelParser_ONNX.hxx
+   TMVA/RModelParser_Keras.h
+   TMVA/RModelParser_PyTorch.h
   SOURCES
     src/RModelParser_ONNX.cxx
     src/ParseBasicUnary.cxx
@@ -80,9 +82,19 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofieParser
     ROOTTMVASofie
 )
 
+# Separate library for the parses that have to link against libpython.
+ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers
+    src/RModelParser_Keras.cxx
+    src/RModelParser_PyTorch.cxx
+  LIBRARIES
+    Python3::NumPy
+    Python3::Python
+    ROOTTMVASofie
+)
+
 target_include_directories(ROOTTMVASofieParser PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
 target_include_directories(ROOTTMVASofieParser BEFORE PUBLIC
   ${Protobuf_INCLUDE_DIRS})
 set_target_properties(ROOTTMVASofieParser PROPERTIES
-  POSITION_INDEPENDENT_CODE TRUE)
+  POSITION_INDEPENDENT_CODE TRUE)

tmva/pymva/inc/TMVA/RModelParser_Keras.h → tmva/sofie_parsers/inc/TMVA/RModelParser_Keras.h

@@ -30,17 +30,11 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyKeras{
-
+namespace TMVA::Experimental::SOFIE::PyKeras {
 
 /// Parser function for translatng Keras .h5 model into a RModel object.
 /// Accepts the file location of a Keras model and returns the
@@ -49,8 +43,6 @@ namespace PyKeras{
 /// has not a defined input batch size : e.g. for input = (input_dim,)
 RModel Parse(std::string filename, int batch_size = -1);
 
-}//PyKeras
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyKeras
+
 #endif //TMVA_PYMVA_RMODELPARSER_KERAS

tmva/pymva/inc/TMVA/RModelParser_PyTorch.h → tmva/sofie_parsers/inc/TMVA/RModelParser_PyTorch.h

@@ -30,31 +30,23 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyTorch{
+namespace TMVA::Experimental::SOFIE::PyTorch {
 
 /// Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model, shapes and data-types of input tensors
 /// and returns the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
 
 /// Overloaded Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model and the shapes of input tensors.
 /// Builds the vector of data-types for input tensors and calls the `Parse()` function to
 /// return the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes);
 
-}//PyTorch
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyTorch
 
 #endif //TMVA_PYMVA_RMODELPARSER_PYTORCH