From 26232636b90231d67c1e6c5cd89d9dfaae9dccb1 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 9 Nov 2025 11:07:36 +0100 Subject: [PATCH 1/3] [tmva][sofie] Disentangle PyMVA and SOFIE So far, the part of SOFIE that uses the C Python API was included in PyMVA for convenience, but the functionality is completely unrelated. Moving this code to SOFIE itself means we can fully build and test SOFIE without enabling `tmva-pymva`. The only subtelty is that we want to be able to disable these SOFIE features that require linking against `libpython`, because this is not always allowed (e.g. in the Python wheels). Therefore, this part is only built if we build also the other ROOT libraries that links against `libpython` besides PyMVA, wich is `TPython`. --- cmake/modules/SearchInstalledSoftware.cmake | 4 +- tmva/pymva/CMakeLists.txt | 5 -- tmva/pymva/inc/LinkDef.h | 4 -- tmva/pymva/test/CMakeLists.txt | 39 +------------ tmva/sofie/test/CMakeLists.txt | 42 ++++++++++++++ .../test/TestRModelParserKeras.C | 18 +++++- .../test/TestRModelParserPyTorch.C | 0 .../test/generateKerasModels.py | 0 .../generatePyTorchModelClassification.py | 0 .../test/generatePyTorchModelMulticlass.py | 0 .../test/generatePyTorchModelRegression.py | 0 .../test/generatePyTorchModels.py | 0 tmva/{pymva => sofie}/test/scale_by_2_op.hxx | 0 tmva/sofie_parsers/CMakeLists.txt | 14 ++++- .../inc/TMVA/RModelParser_Keras.h | 14 +---- .../inc/TMVA/RModelParser_PyTorch.h | 16 ++---- .../src/RModelParser_Keras.cxx | 56 ++++++++++++++----- .../src/RModelParser_PyTorch.cxx | 46 +++++++++++---- tmva/tmva/inc/TMVA/RSofieReader.hxx | 8 +-- 19 files changed, 161 insertions(+), 105 deletions(-) rename tmva/{pymva => sofie}/test/TestRModelParserKeras.C (98%) rename tmva/{pymva => sofie}/test/TestRModelParserPyTorch.C (100%) rename tmva/{pymva => sofie}/test/generateKerasModels.py (100%) rename tmva/{pymva => sofie}/test/generatePyTorchModelClassification.py (100%) rename tmva/{pymva => sofie}/test/generatePyTorchModelMulticlass.py (100%) rename tmva/{pymva => sofie}/test/generatePyTorchModelRegression.py (100%) rename tmva/{pymva => sofie}/test/generatePyTorchModels.py (100%) rename tmva/{pymva => sofie}/test/scale_by_2_op.hxx (100%) rename tmva/{pymva => sofie_parsers}/inc/TMVA/RModelParser_Keras.h (93%) rename tmva/{pymva => sofie_parsers}/inc/TMVA/RModelParser_PyTorch.h (87%) rename tmva/{pymva => sofie_parsers}/src/RModelParser_Keras.cxx (97%) rename tmva/{pymva => sofie_parsers}/src/RModelParser_PyTorch.cxx (96%) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 96ac78ad9b5be..a6dedfbfaf3f8 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva)) elseif(tpython OR tmva-pymva) list(APPEND python_components Development) endif() -if(tmva-pymva) +if(tmva-pymva OR tmva-sofie) list(APPEND python_components NumPy) endif() find_package(Python3 3.9 COMPONENTS ${python_components}) @@ -1796,7 +1796,7 @@ if(tmva) endif() endif() endif() - if(tmva-pymva) + if(tmva-pymva OR tmva-sofie) if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND)) message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required" " (python executable: ${Python3_EXECUTABLE})") diff --git a/tmva/pymva/CMakeLists.txt b/tmva/pymva/CMakeLists.txt index e443405be7440..89ed2de8f754f 100644 --- a/tmva/pymva/CMakeLists.txt +++ b/tmva/pymva/CMakeLists.txt @@ -17,8 +17,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA TMVA/MethodPyKeras.h TMVA/MethodPyRandomForest.h TMVA/MethodPyTorch.h - TMVA/RModelParser_Keras.h - TMVA/RModelParser_PyTorch.h TMVA/PyMethodBase.h SOURCES src/MethodPyAdaBoost.cxx @@ -26,8 +24,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA src/MethodPyKeras.cxx src/MethodPyRandomForest.cxx src/MethodPyTorch.cxx - src/RModelParser_Keras.cxx - src/RModelParser_PyTorch.cxx src/PyMethodBase.cxx LIBRARIES Python3::NumPy @@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA Thread RIO TMVA - ROOTTMVASofie ) ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/tmva/pymva/inc/LinkDef.h b/tmva/pymva/inc/LinkDef.h index 317c124e8e06f..5082dc8949bf0 100644 --- a/tmva/pymva/inc/LinkDef.h +++ b/tmva/pymva/inc/LinkDef.h @@ -15,8 +15,4 @@ #pragma link C++ class TMVA::MethodPyGTB+; #pragma link C++ class TMVA::MethodPyKeras+; #pragma link C++ class TMVA::MethodPyTorch+; -#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras; -#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+; -#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch; -#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+; #endif diff --git a/tmva/pymva/test/CMakeLists.txt b/tmva/pymva/test/CMakeLists.txt index 9605c0bb5e7fa..fb5899fb0fd4f 100644 --- a/tmva/pymva/test/CMakeLists.txt +++ b/tmva/pymva/test/CMakeLists.txt @@ -11,7 +11,7 @@ project(pymva-tests) -set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie) +set(Libraries Core MathCore TMVA PyMVA) # Look for needed python modules ROOT_FIND_PYTHON_MODULE(torch) @@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND) # Enable tests based on available python modules if(ROOT_TORCH_FOUND) - configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY) - configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY) - configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY) - configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY) # Test PyTorch: Binary classification if (ROOT_SKLEARN_FOUND) @@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND) LIBRARIES ${Libraries}) ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends}) - # Test RModelParser_PyTorch - - if(BLAS_FOUND) - ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C - LIBRARIES - ROOTTMVASofie - TMVA - Python3::NumPy - Python3::Python - BLAS::BLAS - INCLUDE_DIRS - SYSTEM - ${CMAKE_CURRENT_BINARY_DIR} - ) - endif() - endif(ROOT_TORCH_FOUND) if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND)) - configure_file(generateKerasModels.py generateKerasModels.py COPYONLY) - configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY) if (ROOT_TORCH_FOUND) set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification) @@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C LIBRARIES ${Libraries}) ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends}) - - if(BLAS_FOUND) - ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C - LIBRARIES - ROOTTMVASofie - PyMVA - Python3::NumPy - Python3::Python - BLAS::BLAS - INCLUDE_DIRS - SYSTEM - ${CMAKE_CURRENT_BINARY_DIR} - ) - endif() - endif() diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index 92fc314fa8e6d..5bf887b760475 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -107,6 +107,8 @@ endif() # Look for needed Python modules ROOT_FIND_PYTHON_MODULE(torch) +ROOT_FIND_PYTHON_MODULE(keras) + if (ROOT_TORCH_FOUND) configure_file(Conv1dModelGenerator.py Conv1dModelGenerator.py COPYONLY) configure_file(Conv2dModelGenerator.py Conv2dModelGenerator.py COPYONLY) @@ -127,6 +129,46 @@ if (ROOT_TORCH_FOUND) endif() endif() +# Any reatures that link against libpython are disabled if built with tpython=OFF +if (tpython AND ROOT_TORCH_FOUND AND BLAS_FOUND) + configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY) + configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY) + configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY) + configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY) + + # Test RModelParser_PyTorch + ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C + LIBRARIES + ROOTTMVASofie + TMVA + Python3::NumPy + Python3::Python + BLAS::BLAS + INCLUDE_DIRS + SYSTEM + ${CMAKE_CURRENT_BINARY_DIR} + ) +endif() + + +# Any reatures that link against libpython are disabled if built with tpython=OFF +if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND) + configure_file(generateKerasModels.py generateKerasModels.py COPYONLY) + configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY) + + ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C + LIBRARIES + ROOTTMVASofie + Python3::NumPy + Python3::Python + BLAS::BLAS + INCLUDE_DIRS + SYSTEM + ${CMAKE_CURRENT_BINARY_DIR} + ) +endif() + + ROOT_EXECUTABLE(emitGNN GNN/EmitGNN.cxx LIBRARIES ROOTTMVASofie) ROOT_ADD_TEST(tmva-sofie-EmitGNN COMMAND emitGNN) diff --git a/tmva/pymva/test/TestRModelParserKeras.C b/tmva/sofie/test/TestRModelParserKeras.C similarity index 98% rename from tmva/pymva/test/TestRModelParserKeras.C rename to tmva/sofie/test/TestRModelParserKeras.C index 69365fd00fe52..3fef4a0bbd9f2 100644 --- a/tmva/pymva/test/TestRModelParserKeras.C +++ b/tmva/sofie/test/TestRModelParserKeras.C @@ -7,10 +7,22 @@ #include "TSystem.h" #include "TMVA/RSofieReader.hxx" -#include "TMVA/PyMethodBase.h" constexpr float DEFAULT_TOLERANCE = 1e-6f; +namespace { + +// Utility functions (taken from PyMethodBase in PyMVA) + +const char *PyStringAsString(PyObject *string) +{ + PyObject *encodedString = PyUnicode_AsUTF8String(string); + const char *cstring = PyBytes_AsString(encodedString); + return cstring; +} + +} // namespace + void GenerateModels() { FILE* fKerasModels = fopen("generateKerasModels.py", "r"); @@ -529,7 +541,7 @@ TEST(RModel, CUSTOM_OP) // get input name for custom (it is output of one before last) PyRun_String("outputName = model.get_layer(index=len(model.layers)-2).output.name",Py_single_input,fGlobalNS,fLocalNS); PyObject *pOutputName = PyDict_GetItemString(fLocalNS, "outputName"); - std::string outputName = TMVA::PyMethodBase::PyStringAsString(pOutputName); + std::string outputName = PyStringAsString(pOutputName); TMVA::Experimental:: RSofieReader r; r.AddCustomOperator(/*OpName*/ "Scale_by_2", /*input tensor names where to insert custom op */std::string("{\"" + outputName + "\"}"), @@ -550,4 +562,4 @@ TEST(RModel, CUSTOM_OP) for (size_t i = 0; i < outputCustomOp.size(); ++i) { EXPECT_LE(std::abs(outputCustomOp[i] - pOutputCustomOp[i]), TOLERANCE); } -} \ No newline at end of file +} diff --git a/tmva/pymva/test/TestRModelParserPyTorch.C b/tmva/sofie/test/TestRModelParserPyTorch.C similarity index 100% rename from tmva/pymva/test/TestRModelParserPyTorch.C rename to tmva/sofie/test/TestRModelParserPyTorch.C diff --git a/tmva/pymva/test/generateKerasModels.py b/tmva/sofie/test/generateKerasModels.py similarity index 100% rename from tmva/pymva/test/generateKerasModels.py rename to tmva/sofie/test/generateKerasModels.py diff --git a/tmva/pymva/test/generatePyTorchModelClassification.py b/tmva/sofie/test/generatePyTorchModelClassification.py similarity index 100% rename from tmva/pymva/test/generatePyTorchModelClassification.py rename to tmva/sofie/test/generatePyTorchModelClassification.py diff --git a/tmva/pymva/test/generatePyTorchModelMulticlass.py b/tmva/sofie/test/generatePyTorchModelMulticlass.py similarity index 100% rename from tmva/pymva/test/generatePyTorchModelMulticlass.py rename to tmva/sofie/test/generatePyTorchModelMulticlass.py diff --git a/tmva/pymva/test/generatePyTorchModelRegression.py b/tmva/sofie/test/generatePyTorchModelRegression.py similarity index 100% rename from tmva/pymva/test/generatePyTorchModelRegression.py rename to tmva/sofie/test/generatePyTorchModelRegression.py diff --git a/tmva/pymva/test/generatePyTorchModels.py b/tmva/sofie/test/generatePyTorchModels.py similarity index 100% rename from tmva/pymva/test/generatePyTorchModels.py rename to tmva/sofie/test/generatePyTorchModels.py diff --git a/tmva/pymva/test/scale_by_2_op.hxx b/tmva/sofie/test/scale_by_2_op.hxx similarity index 100% rename from tmva/pymva/test/scale_by_2_op.hxx rename to tmva/sofie/test/scale_by_2_op.hxx diff --git a/tmva/sofie_parsers/CMakeLists.txt b/tmva/sofie_parsers/CMakeLists.txt index 251f455c658fd..4ad063693fbe5 100644 --- a/tmva/sofie_parsers/CMakeLists.txt +++ b/tmva/sofie_parsers/CMakeLists.txt @@ -21,6 +21,8 @@ set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS -Wno-unused-p ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofieParser HEADERS TMVA/RModelParser_ONNX.hxx + TMVA/RModelParser_Keras.h + TMVA/RModelParser_PyTorch.h SOURCES src/RModelParser_ONNX.cxx src/ParseBasicUnary.cxx @@ -80,9 +82,19 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofieParser ROOTTMVASofie ) +# Separate library for the parses that have to link against libpython. +ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers + src/RModelParser_Keras.cxx + src/RModelParser_PyTorch.cxx + LIBRARIES + Python3::NumPy + Python3::Python + ROOTTMVASofie +) + target_include_directories(ROOTTMVASofieParser PUBLIC $) target_include_directories(ROOTTMVASofieParser BEFORE PUBLIC ${Protobuf_INCLUDE_DIRS}) set_target_properties(ROOTTMVASofieParser PROPERTIES - POSITION_INDEPENDENT_CODE TRUE) \ No newline at end of file + POSITION_INDEPENDENT_CODE TRUE) diff --git a/tmva/pymva/inc/TMVA/RModelParser_Keras.h b/tmva/sofie_parsers/inc/TMVA/RModelParser_Keras.h similarity index 93% rename from tmva/pymva/inc/TMVA/RModelParser_Keras.h rename to tmva/sofie_parsers/inc/TMVA/RModelParser_Keras.h index d60eb2041e650..7e9618306ba74 100644 --- a/tmva/pymva/inc/TMVA/RModelParser_Keras.h +++ b/tmva/sofie_parsers/inc/TMVA/RModelParser_Keras.h @@ -30,17 +30,11 @@ #include "TMVA/Types.h" #include "TMVA/OperatorList.hxx" -#include "TMVA/PyMethodBase.h" - #include "Rtypes.h" #include "TString.h" -namespace TMVA{ -namespace Experimental{ -namespace SOFIE{ -namespace PyKeras{ - +namespace TMVA::Experimental::SOFIE::PyKeras { /// Parser function for translatng Keras .h5 model into a RModel object. /// Accepts the file location of a Keras model and returns the @@ -49,8 +43,6 @@ namespace PyKeras{ /// has not a defined input batch size : e.g. for input = (input_dim,) RModel Parse(std::string filename, int batch_size = -1); -}//PyKeras -}//SOFIE -}//Experimental -}//TMVA +} // namespace TMVA::Experimental::SOFIE::PyKeras + #endif //TMVA_PYMVA_RMODELPARSER_KERAS diff --git a/tmva/pymva/inc/TMVA/RModelParser_PyTorch.h b/tmva/sofie_parsers/inc/TMVA/RModelParser_PyTorch.h similarity index 87% rename from tmva/pymva/inc/TMVA/RModelParser_PyTorch.h rename to tmva/sofie_parsers/inc/TMVA/RModelParser_PyTorch.h index d0cf7914bd4ed..3d02aa9010cb9 100644 --- a/tmva/pymva/inc/TMVA/RModelParser_PyTorch.h +++ b/tmva/sofie_parsers/inc/TMVA/RModelParser_PyTorch.h @@ -30,31 +30,23 @@ #include "TMVA/Types.h" #include "TMVA/OperatorList.hxx" -#include "TMVA/PyMethodBase.h" - #include "Rtypes.h" #include "TString.h" -namespace TMVA{ -namespace Experimental{ -namespace SOFIE{ -namespace PyTorch{ +namespace TMVA::Experimental::SOFIE::PyTorch { /// Parser function for translating PyTorch .pt model into a RModel object. /// Accepts the file location of a PyTorch model, shapes and data-types of input tensors /// and returns the equivalent RModel object. -RModel Parse(std::string filepath,std::vector> inputShapes, std::vector dtype); +RModel Parse(std::string filepath, std::vector> inputShapes, std::vector dtype); /// Overloaded Parser function for translating PyTorch .pt model into a RModel object. /// Accepts the file location of a PyTorch model and the shapes of input tensors. /// Builds the vector of data-types for input tensors and calls the `Parse()` function to /// return the equivalent RModel object. -RModel Parse(std::string filepath,std::vector> inputShapes); +RModel Parse(std::string filepath, std::vector> inputShapes); -}//PyTorch -}//SOFIE -}//Experimental -}//TMVA +} // namespace TMVA::Experimental::SOFIE::PyTorch #endif //TMVA_PYMVA_RMODELPARSER_PYTORCH diff --git a/tmva/pymva/src/RModelParser_Keras.cxx b/tmva/sofie_parsers/src/RModelParser_Keras.cxx similarity index 97% rename from tmva/pymva/src/RModelParser_Keras.cxx rename to tmva/sofie_parsers/src/RModelParser_Keras.cxx index 9bf54dfd8372e..598dbdc18f787 100644 --- a/tmva/pymva/src/RModelParser_Keras.cxx +++ b/tmva/sofie_parsers/src/RModelParser_Keras.cxx @@ -25,16 +25,48 @@ #include -namespace TMVA{ -namespace Experimental{ -namespace SOFIE{ -namespace PyKeras{ +namespace TMVA::Experimental::SOFIE::PyKeras { -// Referencing Python utility functions present in PyMethodBase -static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString; -static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString; -static std::vector(& GetDataFromTuple)(PyObject*) = PyMethodBase::GetDataFromTuple; -static PyObject*(& GetValueFromDict)(PyObject*, const char*) = PyMethodBase::GetValueFromDict; +namespace { + +// Utility functions (taken from PyMethodBase in PyMVA) + +void PyRunString(TString code, PyObject *globalNS, PyObject *localNS) +{ + PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS); + if (!fPyReturn) { + std::cout << "\nPython error message:\n"; + PyErr_Print(); + throw std::runtime_error("\nFailed to run python code: " + code); + } +} + +const char *PyStringAsString(PyObject *string) +{ + PyObject *encodedString = PyUnicode_AsUTF8String(string); + const char *cstring = PyBytes_AsString(encodedString); + return cstring; +} + +std::vector GetDataFromTuple(PyObject *tupleObject) +{ + std::vector tupleVec; + for (Py_ssize_t tupleIter = 0; tupleIter < PyTuple_Size(tupleObject); ++tupleIter) { + auto itemObj = PyTuple_GetItem(tupleObject, tupleIter); + if (itemObj == Py_None) + tupleVec.push_back(0); // case shape is for example (None,2,3) + else + tupleVec.push_back((size_t)PyLong_AsLong(itemObj)); + } + return tupleVec; +} + +PyObject *GetValueFromDict(PyObject *dict, const char *key) +{ + return PyDict_GetItemWithError(dict, PyUnicode_FromString(key)); +} + +} // namespace namespace INTERNAL{ @@ -1036,7 +1068,5 @@ RModel Parse(std::string filename, int batch_size){ return rmodel; } -}//PyKeras -}//SOFIE -}//Experimental -}//TMVA + +} // namespace TMVA::Experimental::SOFIE::PyKeras diff --git a/tmva/pymva/src/RModelParser_PyTorch.cxx b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx similarity index 96% rename from tmva/pymva/src/RModelParser_PyTorch.cxx rename to tmva/sofie_parsers/src/RModelParser_PyTorch.cxx index e97d83e60a35d..34125e5f521d1 100644 --- a/tmva/pymva/src/RModelParser_PyTorch.cxx +++ b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx @@ -28,15 +28,39 @@ #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include -namespace TMVA{ -namespace Experimental{ -namespace SOFIE{ -namespace PyTorch{ +namespace TMVA::Experimental::SOFIE::PyTorch { -// Referencing Python utility functions present in PyMethodBase -static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString; -static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString; -static std::vector(& GetDataFromList)(PyObject*) = PyMethodBase::GetDataFromList; +namespace { + +// Utility functions (taken from PyMethodBase in PyMVA) + +void PyRunString(TString code, PyObject *globalNS, PyObject *localNS) +{ + PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS); + if (!fPyReturn) { + std::cout << "\nPython error message:\n"; + PyErr_Print(); + throw std::runtime_error("\nFailed to run python code: " + code); + } +} + +const char *PyStringAsString(PyObject *string) +{ + PyObject *encodedString = PyUnicode_AsUTF8String(string); + const char *cstring = PyBytes_AsString(encodedString); + return cstring; +} + +std::vector GetDataFromList(PyObject *listObject) +{ + std::vector listVec; + for (Py_ssize_t listIter = 0; listIter < PyList_Size(listObject); ++listIter) { + listVec.push_back((size_t)PyLong_AsLong(PyList_GetItem(listObject, listIter))); + } + return listVec; +} + +} // namespace namespace INTERNAL{ @@ -558,7 +582,5 @@ RModel Parse(std::string filepath,std::vector> inputShapes){ std::vector dtype(inputShapes.size(),ETensorType::FLOAT); return Parse(filepath,inputShapes,dtype); } -}//PyTorch -}//SOFIE -}//Experimental -}//TMVA + +} // namespace TMVA::Experimental::SOFIE::PyTorch diff --git a/tmva/tmva/inc/TMVA/RSofieReader.hxx b/tmva/tmva/inc/TMVA/RSofieReader.hxx index 3d0d2c330eb34..95023df790430 100644 --- a/tmva/tmva/inc/TMVA/RSofieReader.hxx +++ b/tmva/tmva/inc/TMVA/RSofieReader.hxx @@ -110,8 +110,8 @@ public: } else if (type == kKeras) { // use Keras direct parser - if (gSystem->Load("libPyMVA") < 0) { - throw std::runtime_error("RSofieReader: cannot use SOFIE with Keras since libPyMVA is missing"); + if (gSystem->Load("libROOTTMVASofiePyParsers") < 0) { + throw std::runtime_error("RSofieReader: cannot use SOFIE with Keras since libROOTTMVASofiePyParsers is missing"); } // assume batch size is first entry in first input ! std::string batch_size = "-1"; @@ -122,8 +122,8 @@ public: } else if (type == kPt) { // use PyTorch direct parser - if (gSystem->Load("libPyMVA") < 0) { - throw std::runtime_error("RSofieReader: cannot use SOFIE with PyTorch since libPyMVA is missing"); + if (gSystem->Load("libROOTTMVASofiePyParsers") < 0) { + throw std::runtime_error("RSofieReader: cannot use SOFIE with PyTorch since libROOTTMVASofiePyParsers is missing"); } if (inputShapes.size() == 0) { throw std::runtime_error("RSofieReader: cannot use SOFIE with PyTorch since the input tensor shape is missing and is needed by the PyTorch parser"); From 09d419d5b02bc02ff15692becbdbbf2f94b6b37a Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 9 Nov 2025 14:23:05 +0100 Subject: [PATCH 2/3] [tmva][sofie] Remove `batch_size` keyword arument from Dense layers This keyword argument is not relevant for defining the model and will cause errors in newer Keras versions. --- tmva/sofie/test/generateKerasModels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tmva/sofie/test/generateKerasModels.py b/tmva/sofie/test/generateKerasModels.py index 39872f2d38c6c..7fc05741662aa 100644 --- a/tmva/sofie/test/generateKerasModels.py +++ b/tmva/sofie/test/generateKerasModels.py @@ -28,7 +28,7 @@ def generateFunctionalModel(): def generateSequentialModel(): model=Sequential() - model.add(Dense(8,batch_size=4)) + model.add(Dense(8)) model.add(ReLU()) model.add(Dense(6)) model.add(Activation('sigmoid')) @@ -43,7 +43,7 @@ def generateSequentialModel(): def generateBatchNormModel(): model=Sequential() - model.add(Dense(4,batch_size=2)) + model.add(Dense(4)) model.add(BatchNormalization()) model.add(Dense(2)) @@ -180,4 +180,4 @@ def generateCustomModel(): generateBinaryOpModel() generateActivationModel() generateSwishModel() -generateCustomModel() \ No newline at end of file +generateCustomModel() From 4b863e2b047e40551bf69ef19332d6a8b8c4067f Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 9 Nov 2025 15:21:26 +0100 Subject: [PATCH 3/3] [tmva][sofie] Don't run Keras tests if the Keras version is too new --- tmva/sofie/test/CMakeLists.txt | 36 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index 5bf887b760475..50915da185643 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -153,19 +153,29 @@ endif() # Any reatures that link against libpython are disabled if built with tpython=OFF if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND) - configure_file(generateKerasModels.py generateKerasModels.py COPYONLY) - configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY) - - ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C - LIBRARIES - ROOTTMVASofie - Python3::NumPy - Python3::Python - BLAS::BLAS - INCLUDE_DIRS - SYSTEM - ${CMAKE_CURRENT_BINARY_DIR} - ) + + set(unsupported_keras_version "3.10.0") + + # TODO: make sure we also support the newest Keras + if (NOT DEFINED ROOT_KERAS_VERSION) + message(WARNING "Keras found, but version unknown — cannot verify compatibility.") + elseif (ROOT_KERAS_VERSION VERSION_LESS ${unsupported_keras_version}) + configure_file(generateKerasModels.py generateKerasModels.py COPYONLY) + configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY) + + ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C + LIBRARIES + ROOTTMVASofie + Python3::NumPy + Python3::Python + BLAS::BLAS + INCLUDE_DIRS + SYSTEM + ${CMAKE_CURRENT_BINARY_DIR} + ) + else() + message(WARNING "Keras version ${ROOT_KERAS_VERSION} is too new for the SOFIE Keras parser (only supports < ${unsupported_keras_version})") + endif() endif()