Skip to content

Commit 4412897

Browse files
AlexanderViand-IntelkylaneraceAlexanderViandjoserochh
authored
Tracing Data Formats (#111)
* Adding First Version of Data Formats Library * data-formats: various fixes and improvements * update data_formats to new format/pylint/etc * remove need to run cmake for python protobuf tests * fix cpplint issues * fix issues flagged by warnings indicating legitimate issues * data-formats: add `store/load_json_trace` * add openfhe tracer (w/ end-to-end example) * program_mapper: add support for `muli` * nit: fix leftover #endif w/o #if * data_formats: print prettier json * add some info output to run_tracing_example * populate missing metadata * nit: clang-format Note: `run_tracing_example` target works up to expected functional modeler issue `key not found: partQHatInvModq_0_0` which is due to the fact that partQHatInvModq is no longer present in recent OpenFHE versions. --------- Co-authored-by: Race, Kylan E <kylan.e.race@intel.com> Co-authored-by: Alexander Viand <alexander.viand@intel.com> Co-authored-by: Jose Rojas Chaves <jose.rojas.chaves@intel.com>
1 parent 52fbecf commit 4412897

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+7436
-22
lines changed

‎.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,3 +204,11 @@ dmypy.json
204204

205205
# Pyre type checker
206206
.pyre/
207+
208+
# Generated protobuf Python files (auto-generated by tests)
209+
p-isa_tools/data_formats/python/heracles/proto/*_pb2.py
210+
!p-isa_tools/data_formats/python/heracles/proto/__init__.py
211+
212+
# Test artifacts
213+
p-isa_tools/data_formats/test/*.program_trace
214+
p-isa_tools/data_formats/test/*.data_trace*

‎.typos.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
# variation of params
99
parms = "parms"
1010
bload = "bload"
11+
ser = "ser"
12+
SerType = "SerType"
1113

1214
[files]
1315
extend-exclude = [

‎p-isa_tools/CPPLINT.cfg‎ renamed to ‎CPPLINT.cfg‎

File renamed without changes.

‎p-isa_tools/CMakeLists.txt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,15 @@ else()
2222
set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of Build" FORCE)
2323
endif()
2424

25-
option(ENABLE_DATA_FORMATS "Enable support for the data formats library" OFF)
25+
option(ENABLE_DATA_FORMATS "Enable support for the data formats library" ON)
2626
message(ENABLE_DATA_FORMATS="${ENABLE_DATA_FORMATS}")
2727

2828
option(ENABLE_FUNCTIONAL_MODELER "Enable building of functional modeler" ON)
2929
message(ENABLE_FUNCTIONAL_MODELER="${ENABLE_FUNCTIONAL_MODELER}")
3030

31+
option(ENABLE_KERNGEN "Enable kerngen (dependencies only)" ON)
32+
message(ENABLE_KERNGEN="${ENABLE_KERNGEN}")
33+
3134
option(ENABLE_PROGRAM_MAPPER "Enable building of program mapper" ON)
3235
message(ENABLE_PROGRAM_MAPPER="${ENABLE_PROGRAM_MAPPER}")
3336

@@ -63,6 +66,12 @@ file(GLOB_RECURSE IDE_HEADERS program_mapper/*.h functional_modeler/*.h dependen
6366

6467
# Build sub-directories
6568
add_subdirectory(common)
69+
if(ENABLE_KERNGEN)
70+
add_subdirectory(kerngen)
71+
endif()
72+
if(ENABLE_DATA_FORMATS)
73+
add_subdirectory(data_formats)
74+
endif()
6675
if(ENABLE_FUNCTIONAL_MODELER)
6776
add_subdirectory(functional_modeler)
6877
endif()

‎p-isa_tools/cmake/dependencies.cmake‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,3 @@ if (NOT snap_POPULATED)
4545
include_directories(${snap_SOURCE_DIR}/snap-core ${snap_SOURCE_DIR}/glib-core)
4646
message(STATUS "Finished building SNAP")
4747
endif()
48-
49-
if(ENABLE_DATA_FORMATS)
50-
find_package(HERACLES_DATA_FORMATS CONFIG)
51-
if(NOT HERACLES_DATA_FORMATS_FOUND)
52-
FetchContent_Declare(
53-
heracles_data_formats
54-
GIT_REPOSITORY git@github.com:IntelLabs/HERACLES-data-formats.git
55-
GIT_TAG main
56-
)
57-
FetchContent_MakeAvailable(heracles_data_formats)
58-
endif()
59-
endif()

‎p-isa_tools/data_formats/CMakeLists.txt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
cmake_minimum_required(VERSION 3.15.0...3.24.0)
2+
3+
project(HERACLES_DATA_FORMATS VERSION 1.0.0)
4+
5+
set(CMAKE_CXX_STANDARD 17)
6+
set(HERACLES_DATA_FORMATS_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
7+
include(CMakePackageConfigHelpers)
8+
include(GNUInstallDirs)
9+
10+
# Set default output directories
11+
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
12+
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
13+
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}")
14+
15+
include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake)
16+
include(${PROJECT_SOURCE_DIR}/cmake/protobuf.cmake)
17+
18+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -fPIC")
19+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -fPIC")
20+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
21+
set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
22+
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_INSTALL_LIBDIR})
23+
24+
find_package(OpenMP REQUIRED)
25+
26+
option(BUILD_TEST "Build c++/python tests with ctest" ON)
27+
enable_testing()
28+
29+
add_subdirectory(proto)
30+
add_subdirectory(cpp)
31+
if(BUILD_TEST)
32+
add_subdirectory(test)
33+
endif()
34+
35+
# install python utility functions
36+
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/
37+
DESTINATION python
38+
# TODO(skmono): to be added afterwards
39+
# FILES_MATCHING
40+
# PATTERN "*.py"
41+
)
42+
43+
# copy python utility functions to build/python
44+
add_custom_target(HERACLES_DATA_FORMATS_COPY_PYTHON
45+
ALL
46+
DEPENDS HERACLES_data_proto
47+
)
48+
add_custom_command(
49+
TARGET HERACLES_DATA_FORMATS_COPY_PYTHON
50+
POST_BUILD
51+
COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/python ${PROJECT_BINARY_DIR}/python/
52+
)
53+
54+
option(ENABLE_OPENFHE_TRACER "Build the OpenFHE tracer" ON)
55+
if(ENABLE_OPENFHE_TRACER)
56+
add_subdirectory(tracers/openfhe)
57+
endif()

‎p-isa_tools/data_formats/README.md‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# HERACLES Data formatter interface
2+
3+
## CMake Configure and Build
4+
```bash
5+
cmake -S . -B build
6+
cmake --build build --parallel
7+
```
8+
_Note: for now cmake will _not build with `ninja`_ and is only tested for
9+
(default) `CMAKE_GENERATOR='Unix Makefiles`_
10+
11+
12+
## Run test
13+
```bash
14+
cmake --build build --target test
15+
```
16+
17+
Note: Python `[dev]` dependencies from the root `pyproject.toml` are required to run the Python test. They can be installed via
18+
```bash
19+
pip install -e ".[dev]" # from repository root
20+
```
21+
22+
## C++
23+
24+
### Importing the **HERACLES-Data-Formats** Library
25+
26+
The C++ library be found and included with cmake by including
27+
following statements in the cmakefile of the project depending on the
28+
HERACLES data formats library:
29+
```cmake
30+
find_package(HERACLES_DATA_FORMATS 1.0.0 REQUIRED)
31+
...
32+
target_link_libraries(<YOUR_LIBRARY> PUBLIC HERACLES_DATA_FORMATS::heracles_data_formats)
33+
```
34+
Assuming you follow the convention of having all code
35+
checked out in the same directory and named by their component name, you
36+
can then build that project by executing the following:
37+
38+
```bash
39+
# from project root
40+
HERACLES_DATA_FORMATS_DIR=$(pwd)/../HERACLES-data-formats/build cmake -S . -B build
41+
cmake --build build --parallel
42+
```
43+
Alternatively, you can also build and install HERACLES-data-formats
44+
(with the destination chosen, e.g., using the
45+
`-DCMAKE_INSTALL_PREFIX=/path/to/install` argument, and an `cmake
46+
--build build --target install` after the build ). However, when
47+
installing be careful in not forgetting to re-install
48+
after each change and subsequent build or accidentally picking up
49+
older versions installed elsewhere and earlier searched in CMAKE's
50+
search paths.
51+
52+
53+
### Usage example
54+
The library can be used in the ```C++``` code, e.g., as followed:
55+
```c++
56+
// protobuf headers
57+
#include "heracles/heracles_proto.h"
58+
// cpp utility headers
59+
#include "heracles/heracles_data_formats.h"
60+
61+
int main() {
62+
heracles::fhe_trace::Trace trace;
63+
heracles::data::InputPolynomials input_polys;
64+
65+
return 0;
66+
}
67+
```
68+
Refer to the [heracles_test.cpp](src/data_formats/test/heracles_test.cpp) source
69+
code for additional examples of using Heracles protobuf objects and
70+
utility functions as well as [Protocol Buffer Basics:
71+
C++](https://protobuf.dev/getting-started/cpptutorial/) for more
72+
general information on using generated C++ protobuf code.
73+
74+
75+
## Python
76+
77+
78+
For the Python package to be used independently of CMake/C++ builds, the optional `dev` dependencies are required.
79+
80+
1. **Install dependencies**:
81+
```bash
82+
# For development (includes grpcio-tools for compiling protos, pytest for testing)
83+
pip install -e ".[dev]"
84+
```
85+
86+
2. **Compile Protocol Buffers**:
87+
```bash
88+
python p-isa_tools/data_formats/compile_protos.py
89+
```
90+
91+
This generates the Python protobuf files in `p-isa_tools/data_formats/python/heracles/proto/`.
92+
93+
3. **Generate test traces** (if needed for testing):
94+
```bash
95+
python p-isa_tools/data_formats/test/generate_test_traces.py
96+
```
97+
Alternatively, you can simply run the `pytest` tests, which will create the protobuf files and/or test traces if they do not exist yet.
98+
99+
### Running Tests
100+
101+
From the repository root:
102+
```bash
103+
pytest p-isa_tools/data_formats/test/
104+
```
105+
(The path is optional, but avoids running unrelated tests)
106+
107+
### Usage example
108+
The **HERACLES-Data-Formats** library can be imported via, e.g.,
109+
```python
110+
from heracles.proto.common_pb2 import Scheme
111+
from heracles.proto.fhe_trace_pb2 import Trace, Instruction
112+
import heracles.fhe_trace.io as hfi
113+
import heracles.data.io as hdi
114+
115+
# Create and save a trace
116+
trace = Trace()
117+
trace.scheme = Scheme.SCHEME_BGV
118+
hfi.store_trace("my_trace.bin", trace)
119+
120+
# Load a trace
121+
loaded_trace = hfi.load_trace("my_trace.bin")
122+
```
123+
124+
Refer to the [heracles_test.py](test/heracles_test.py) script for
125+
examples of using Heracles protobuf objects and utility functions as
126+
well as [Protocol Buffer Basics:
127+
Python](https://protobuf.dev/getting-started/pythontutorial/) for more
128+
general information on using generated python protobuf code.

‎p-isa_tools/data_formats/cmake/HERACLES_DATA_FORMATSConfig.cmake.in‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright (C) 2025 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
@PACKAGE_INIT@
5+
6+
include(CMakeFindDependencyMacro)
7+
8+
include(${CMAKE_CURRENT_LIST_DIR}/HERACLES_DATA_FORMATSTargets.cmake)
9+
10+
if(TARGET HERACLES_DATA_FORMATS::heracles_data_formats)
11+
set(HERACLES_DATA_FORMATS_FOUND TRUE)
12+
message(STATUS "Heracles Data Formats Library found")
13+
else()
14+
message(STATUS "Heracles Data Formats Library not found")
15+
endif()
16+
17+
# Requirement for protobuf
18+
find_package(ZLIB REQUIRED)
19+
20+
set(HERACLES_DATA_FORMATS_VERSION "@HERACLES_DATA_FORMATS_VERSION")
21+
set(HERACLES_DATA_FORMATS_VERSION_MAJOR "@HERACLES_DATA_FORMATS_VERSION_MAJOR")
22+
set(HERACLES_DATA_FORMATS_VERSION_MINOR "@HERACLES_DATA_FORMATS_VERSION")
23+
set(HERACLES_DATA_FORMATS_VERSION_PATCH "@HERACLES_DATA_FORMATS_VERSION")
24+
25+
set(HERACLES_DATA_FORMATS_DEBUG "@HERACLES_DATA_FORMATS_DEBUG")

0 commit comments

Comments
 (0)