diff --git a/.github/actions/build-py/action.yml b/.github/actions/build-py/action.yml index 2cb89fa75d..e0632831b3 100644 --- a/.github/actions/build-py/action.yml +++ b/.github/actions/build-py/action.yml @@ -19,9 +19,11 @@ runs: shell: bash run: | cd pycode/memilio-${{ inputs.package }}/ - /opt/python/cp38-cp38/bin/python -m pip install scikit-build + /opt/python/cp38-cp38/bin/python -m pip install --upgrade pip setuptools wheel + /opt/python/cp38-cp38/bin/python -m pip install scikit-build scikit-build-core /opt/python/cp38-cp38/bin/python -m build --no-isolation --wheel - /opt/python/cp311-cp311/bin/python -m pip install scikit-build + /opt/python/cp311-cp311/bin/python -m pip install --upgrade pip setuptools wheel + /opt/python/cp311-cp311/bin/python -m pip install scikit-build scikit-build-core /opt/python/cp311-cp311/bin/python -m build --no-isolation --wheel # Exclude memilio-generation, because its a pure python package, cmake is only used in the build process to retrieve data from cpp if [[ -f "CMakeLists.txt" ]] && [ "${{ inputs.package }}" != "generation" ]; then diff --git a/.github/actions/test-pylint/action.yml b/.github/actions/test-pylint/action.yml index 9f9f50bc2c..021e3a0471 100644 --- a/.github/actions/test-pylint/action.yml +++ b/.github/actions/test-pylint/action.yml @@ -13,6 +13,10 @@ runs: sudo apt-get -qq update sudo apt-get -qq -y install python3-pip gnupg python -m pip install --upgrade pip + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" - name: Download Python Wheels uses: actions/download-artifact@v4 with: @@ -28,7 +32,11 @@ runs: run: | cd pycode/memilio-${{ inputs.package }} mkdir -p build_pylint - python setup.py pylint + if [ -f tools/run_pylint.py ]; then + python tools/run_pylint.py + else + python -m pylint --rcfile=../pylintrc --load-plugins=pylint_json2html --output-format=jsonextended memilio/ > build_pylint/pylint_extended.json || true + fi pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json - name: Upload Pylint Report uses: actions/upload-artifact@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 859c882a57..3f637630d7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ build: jobs: install: - pip install --exists-action=w --no-cache-dir -r docs/requirements-build.txt - - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt --no-build-isolation + - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt sphinx: configuration: docs/source/conf.py diff --git a/cpp/thirdparty/CMakeLists.txt b/cpp/thirdparty/CMakeLists.txt index 1fc70bfdc3..68d9904ac0 100644 --- a/cpp/thirdparty/CMakeLists.txt +++ b/cpp/thirdparty/CMakeLists.txt @@ -37,7 +37,22 @@ if(MEMILIO_USE_BUNDLED_SPDLOG) if(NOT spdlog_POPULATED) FetchContent_Populate(spdlog) + + # Even when the rest builds shared libraries, we sometimes + # prefer to keep spdlog static (e.g. for Python wheels where we don't want + # an extra shared dependency). Temporarily override BUILD_SHARED_LIBS while + # configuring spdlog to force a static build in that case. + if(MEMILIO_PREFER_STATIC_SPDLOG) + set(_MEMILIO_PREV_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS OFF) + endif() + add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR} EXCLUDE_FROM_ALL) + + if(MEMILIO_PREFER_STATIC_SPDLOG) + set(BUILD_SHARED_LIBS ${_MEMILIO_PREV_BUILD_SHARED_LIBS}) + unset(_MEMILIO_PREV_BUILD_SHARED_LIBS) + endif() endif() else() @@ -198,6 +213,7 @@ else() endif() find_package(sbml ${MEMILIO_SBML_VERSION} CONFIG) + if(sbml_FOUND) set(MEMILIO_HAS_SBML ON) message(STATUS "Found libSBML: ${libsbml_INCLUDE_DIRS}") diff --git a/docs/source/python/m-epidata.rst b/docs/source/python/m-epidata.rst index c908e339c9..8c306d90c7 100644 --- a/docs/source/python/m-epidata.rst +++ b/docs/source/python/m-epidata.rst @@ -85,7 +85,7 @@ When you start creating a new script: When you add a new script -- Add an executable to the setup.py in "pycode/memilio-epidata". +- Add a console script entry to the ``pycode/memilio-epidata/pyproject.toml`` file. - Add it to the cli_dict in getDataIntoPandasDataFrame.py. - Add a meaningful key for the new script. - for the dict value add a list in the form [comment to print when script is started, list of used parser arguments (optional)]. diff --git a/docs/source/python/m-generation.rst b/docs/source/python/m-generation.rst index 858edd92d4..bcabac83f1 100644 --- a/docs/source/python/m-generation.rst +++ b/docs/source/python/m-generation.rst @@ -30,7 +30,7 @@ For a successful build, the development libraries for Python need to be installe .. warning:: Generation currently requires specifically version ``18.1.1`` of `libclang.so`, since the function ``create_ast`` in `ast.py `_ generates the abstract syntax tree using `clang-18`. Different versions may lead to unsupported abstractions. - If you want to try a different version, set your `libclang` version under ``install_requires`` in the `setup.py `_ and change the clang command in ``create_ast`` in `ast.py `_. + If you want to try a different version, set your `libclang` version under ``dependencies`` in the `pyproject.toml `_ and change the clang command in ``create_ast`` in `ast.py `_. Usage ----- diff --git a/docs/source/python/python_packages.rst b/docs/source/python/python_packages.rst index e9c703a429..daa5ed893d 100644 --- a/docs/source/python/python_packages.rst +++ b/docs/source/python/python_packages.rst @@ -111,9 +111,9 @@ Please see the individual package documentation for more details on the function Installation ------------ -Each package provides a `setup.py` script that installs the package and its dependencies. +Each package provides a ``pyproject.toml`` that installs the package and its dependencies with pip. The dependencies of the individual packages are denoted in their documentation. -The installation can be run with the following command (from the directory containing the `setup.py`) +The installation can be run with the following command (from the directory containing the ``pyproject.toml`` file) .. code-block:: console @@ -171,9 +171,11 @@ Run pylint with the commands in the package folder .. code-block:: console - python setup.py pylint + python tools/run_pylint.py pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json +Packages without the helper script can run pylint directly, for example ``python -m pylint memilio``. + Pylint report for actual master: -`Pylint Report `_ \ No newline at end of file +`Pylint Report `_ diff --git a/pycode/memilio-epidata/README.rst b/pycode/memilio-epidata/README.rst index be21a21a89..69a9a692e3 100644 --- a/pycode/memilio-epidata/README.rst +++ b/pycode/memilio-epidata/README.rst @@ -23,9 +23,9 @@ A more detailed description of the sources can be found in the `epidata subfolde Installation ------------ -Use the provided ``setup.py`` script to install the package and its dependencies. +This project uses ``pyproject.toml`` to install the package and its dependencies. -To install the package, use (from the directory that contains ``setup.py``) +To install the package, run (from the directory that contains ``pyproject.toml``) .. code:: sh @@ -116,7 +116,7 @@ Run pylint with the commands .. code:: sh - python setup.py pylint + python tools/run_pylint.py pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json Pylint report for actual master: @@ -163,7 +163,7 @@ When you start creating a new script: When you add a new script -- add a executable to the setup.py in "pycode/memilio-epidata" +- add a script entry to ``pyproject.toml`` in "pycode/memilio-epidata" - add it to the cli_dict in getDataIntoPandasDataFrame.py - add a meaningfull key for the new script - as the value add a list in the form [comment to print when script is started, list of used parser arguments (optional)] diff --git a/pycode/memilio-epidata/pyproject.toml b/pycode/memilio-epidata/pyproject.toml new file mode 100644 index 0000000000..b282267810 --- /dev/null +++ b/pycode/memilio-epidata/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "memilio-epidata" +version = "1.0.0" +description = "Part of MEmilio project, reads epidemiological data from different official and unofficial sources." +readme = "README.rst" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "MEmilio Team" }] +maintainers = [ + { email = "martin.kuehn@dlr.de" } +] +dependencies = [ + "pandas>=2.0.0", + "pyarrow", + "matplotlib", + "tables", + "numpy>=1.22,<1.25", + "openpyxl", + "xlrd", + "xlsxwriter", + "requests", + "pyxlsb", + "wget", + "twill==3.1", + "PyQt6-sip<13.9", + "PyQt6", + "python-calamine", + "python-magic-bin; sys_platform == 'win32'", + "python-magic; sys_platform != 'win32'" +] + +[project.optional-dependencies] +dev = [ + "pyfakefs>=4.6,<5.3.3", + "coverage>=7.0.1", + "pylint>=2.13.0,<2.16", + "pylint_json2html==0.4.0" +] + +[project.scripts] +getcasedata = "memilio.epidata.getCaseData:main" +getpopuldata = "memilio.epidata.getPopulationData:main" +getjhdata = "memilio.epidata.getJHData:main" +getdividata = "memilio.epidata.getDIVIData:main" +getsimdata = "memilio.epidata.getSimulationData:main" +cleandata = "memilio.epidata.cleanData:main" +getcommutermobility = "memilio.epidata.getCommuterMobility:main" +getvaccinationdata = "memilio.epidata.getVaccinationData:main" +gethospitalizationdata = "memilio.epidata.getHospitalizationData:main" + +[project.urls] +Homepage = "https://github.com/SciCompMod/memilio" +Team = "https://memilio.readthedocs.io/en/latest/team.html" + +[tool.setuptools.packages.find] +where = ["."] +include = ["memilio*"] diff --git a/pycode/memilio-epidata/setup.py b/pycode/memilio-epidata/setup.py deleted file mode 100644 index 3d6616cd40..0000000000 --- a/pycode/memilio-epidata/setup.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import subprocess -import sys - -from setuptools import Command, find_packages, setup - -__version__ = '1.0.0' - - -class PylintCommand(Command): - """Custom command to run pylint and get a report as html.""" - description = "Runs pylint and outputs the report as html." - user_options = [] - - def initialize_options(self): - """ """ - from pylint.reporters.json_reporter import JSONReporter - from pylint.reporters.text import ParseableTextReporter, TextReporter - from pylint_json2html import JsonExtendedReporter - - self.lint_modules = ["memilio/"] - self.out_format = "extendedjson" - - self.REPORTERS = { - "parseable": (ParseableTextReporter, "build_pylint/pylint_parseable.txt"), - "text": (TextReporter, "build_pylint/pylint.txt"), - "json": (JSONReporter, "build_pylint/pylint.json"), - "extendedjson": (JsonExtendedReporter, "build_pylint/pylint_extended.json") - } - - def finalize_options(self): - """ """ - self.reporter, self.out_file = self.REPORTERS.get( - self.out_format) # , self.REPORTERS.get("parseable")) - - def run(self): - """ """ - os.makedirs("build_pylint", exist_ok=True) - - # Run pylint - from pylint import lint - with open(self.out_file, "w", encoding="utf-8") as report_file: - options = ["--rcfile=../pylintrc", *self.lint_modules] - - lint.Run(options, reporter=self.reporter( - report_file), do_exit=False) - - -# Python-magic needs DLLs for libmagic. They have to be installed only on windows. -if sys.platform == 'win32': - pymagic = 'python-magic-bin' -else: - pymagic = 'python-magic' - -setup( - name='memilio-epidata', - version=__version__, - author='DLR-SC', - author_email='daniel.abele@dlr.de', - maintainer_email='martin.kuehn@dlr.de', - url='https://github.com/SciCompMod/memilio', - description='Part of MEmilio project, reads epidemiological data from different official and unofficial sources.', - entry_points={ - 'console_scripts': [ - 'getcasedata=memilio.epidata.getCaseData:main', - 'getpopuldata=memilio.epidata.getPopulationData:main', - 'getjhdata = memilio.epidata.getJHData:main', - 'getdividata = memilio.epidata.getDIVIData:main', - 'getsimdata = memilio.epidata.getSimulationData:main', - 'cleandata = memilio.epidata.cleanData:main', - 'getcommutermobility = memilio.epidata.getCommuterMobility:main', - 'getvaccinationdata = memilio.epidata.getVaccinationData:main', - 'gethospitalizationdata = memilio.epidata.getHospitalizationData:main' - ], - }, - packages=find_packages(where=os.path.dirname(os.path.abspath(__file__))), - long_description='', - test_suite='memilio.epidata_test', - install_requires=[ - # pandas 2.0 is minimum for CoW - 'pandas>=2.0.0', - # FutureWarning of pandas that pyarrow will be required in a future release - 'pyarrow', - 'matplotlib', - 'tables', - # smaller numpy versions cause a security issue, 1.25 breaks testing with pyfakefs - 'numpy>=1.22,<1.25', - 'openpyxl', - 'xlrd', - 'xlsxwriter', - 'requests', - 'pyxlsb', - 'wget', - 'twill==3.1', - # set PyQt6-sip version as the one pulled by PyQt6 in Epidata-CI (using manylinux_2_28_x86_64) req. python 3.9 - 'PyQt6-sip<13.9', - 'PyQt6', - 'python-calamine', - pymagic - ], - extras_require={ - 'dev': [ - # first support of python 3.11 4.6 - # 5.3.4 has conflicts with openpyxl - # 5.3.3 broken - 'pyfakefs>=4.6,<5.3.3', - # coverage 7.0.0 can't find .whl files and breaks CI - 'coverage>=7.0.1', - # pylint 2.16 creates problem with wrapt package version - 'pylint>=2.13.0,<2.16', - 'pylint_json2html==0.4.0', - ], - }, - cmdclass={ - 'pylint': PylintCommand - }, -) diff --git a/pycode/memilio-epidata/tools/run_pylint.py b/pycode/memilio-epidata/tools/run_pylint.py new file mode 100644 index 0000000000..2f9b4b7c2c --- /dev/null +++ b/pycode/memilio-epidata/tools/run_pylint.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Generate Pylint reports in jsonextended format for the epidata package.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def main() -> int: + project_dir = Path(__file__).resolve().parent.parent + repo_root = project_dir.parent + build_dir = project_dir / "build_pylint" + build_dir.mkdir(exist_ok=True) + output_file = build_dir / "pylint_extended.json" + + cmd = [ + sys.executable, + "-m", + "pylint", + "--rcfile", + str(repo_root / "pylintrc"), + "--load-plugins", + "pylint_json2html", + "--output-format=jsonextended", + "memilio/", + ] + + with output_file.open("w", encoding="utf-8") as stream: + result = subprocess.run( + cmd, + cwd=project_dir, + stdout=stream, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + if result.stderr: + sys.stderr.write(result.stderr) + + if result.returncode: + sys.stderr.write( + "pylint reported issues; see build_pylint/pylint_extended.json for details.\n" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pycode/memilio-generation/README.md b/pycode/memilio-generation/README.md index 31ae30a0c2..b8ebedb7e0 100644 --- a/pycode/memilio-generation/README.md +++ b/pycode/memilio-generation/README.md @@ -13,7 +13,7 @@ The package uses the [Clang C++ library](https://clang.llvm.org/) and the [LibCl ## Installation -Use the provided `setup.py` script to build and install the package. To install the package, use the command (from the directory containing `setup.py`) +Use the provided `pyproject.toml` file to build and install the package. To install the package, use the command (from the directory containing `pyproject.toml`) ```bash pip install -e .[dev] @@ -51,4 +51,4 @@ When implementing new model features you can follow these steps: - Adjust the substitution dictionaries in the [Generator class](./memilio/generation/generator.py). - Write new/Adjust script in the [tool folder](./memilio/tools/) for the model and try to run. - Add new strings in the [Default dict](/pycode/memilio-generation/memilio/generation/default_generation_dict.py) -- Update [tests](./memilio/generation_test/). \ No newline at end of file +- Update [tests](./memilio/generation_test/). diff --git a/pycode/memilio-generation/memilio/generation/scanner_config.py b/pycode/memilio-generation/memilio/generation/scanner_config.py index a4fb482133..a52a31244b 100644 --- a/pycode/memilio-generation/memilio/generation/scanner_config.py +++ b/pycode/memilio-generation/memilio/generation/scanner_config.py @@ -39,7 +39,7 @@ class ScannerConfig: path_database: Path to the folder of the compile_commands.json namespace: C++ namespace of the model class python_module_name: Individual name for binded python module - python_generation_module_path: Path to the setup.py of the generation module + python_generation_module_path: Path to the ``pyproject.toml`` of the generation module skbuild_path_to_database: Path to compile_commands.json target_folder: Target folder for generated files optional: List with optional arguments diff --git a/pycode/memilio-generation/pyproject.toml b/pycode/memilio-generation/pyproject.toml new file mode 100644 index 0000000000..12cb7ed1e0 --- /dev/null +++ b/pycode/memilio-generation/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = [ + "scikit-build-core>=0.9.0", + "setuptools>=68", + "wheel" +] +build-backend = "scikit_build_core.build" + +[project] +name = "memilio-generation" +version = "0.1.0" +description = "Part of MEmilio project, automatic generation of model specific python bindings." +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "MEmilio Team" }] +maintainers = [ + { email = "martin.kuehn@dlr.de" } +] +dependencies = [ + "libclang==18.1.1", + "dataclasses", + "dataclasses_json", + "graphviz", + "importlib-resources>=1.1.0; python_version < '3.9'" +] + +[project.optional-dependencies] +dev = [] + +[project.urls] +Homepage = "https://github.com/SciCompMod/memilio" +Team = "https://memilio.readthedocs.io/en/latest/team.html" + +[tool.scikit-build] +cmake.version = ">=3.13" +wheel.packages = ["memilio"] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.package-data] +"memilio.generation" = ["tools/*.json", "tools/*.txt", "tools/README.md"] diff --git a/pycode/memilio-generation/setup.py b/pycode/memilio-generation/setup.py deleted file mode 100644 index 2d3370d80e..0000000000 --- a/pycode/memilio-generation/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import subprocess -import sys - -from setuptools import find_packages, setup - -try: - from skbuild import setup -except ImportError: - print('scikit-build is required to build from source.') - print('Installation: python -m pip install scikit-build') - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "scikit-build"]) - from skbuild import setup - -__version__ = '0.1.0' - -setup( - name='memilio-generation', - version=__version__, - author='DLR-SC', - author_email='maximilian.betz@dlr.de', - maintainer_email='martin.kuehn@dlr.de', - url='https://github.com/SciCompMod/memilio', - description='Part of MEmilio project, automatic generation of model specific python bindings.', - packages=find_packages( - where=os.path.dirname(os.path.abspath(__file__))), - install_requires=['libclang==18.1.1', - 'dataclasses', 'dataclasses_json', 'graphviz', 'importlib-resources>=1.1.0; python_version < \'3.9\''], - extras_require={'dev': []}, - long_description='', - test_suite='memilio.generation_test', - package_data={'memilio.generation': [ - '../../_skbuild/*/cmake-build/compile_commands.json', '../tools/config.json']}, -) diff --git a/pycode/memilio-plot/README.md b/pycode/memilio-plot/README.md index 014b8a8221..97d0d06a42 100644 --- a/pycode/memilio-plot/README.md +++ b/pycode/memilio-plot/README.md @@ -23,9 +23,9 @@ by other packages of the MEmilio software. Installation ------------ -Use the provided ``setup.py`` script to install the package and its dependencies. +This project uses ``pyproject.toml`` to install the package and its dependencies. -To install the package, use (from the directory that contains ``setup.py``) +To install the package, use pip (from the directory that contains ``pyproject.toml``) .. code:: sh @@ -104,7 +104,7 @@ Run pylint with the commands .. code:: sh - python setup.py pylint + python tools/run_pylint.py pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json Pylint report for actual master: diff --git a/pycode/memilio-plot/pyproject.toml b/pycode/memilio-plot/pyproject.toml new file mode 100644 index 0000000000..aab85cbb6a --- /dev/null +++ b/pycode/memilio-plot/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "memilio-plot" +version = "1.0.0" +description = "Part of MEmilio project, plots data to maps or visualizes simulation curves." +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "MEmilio Team" }] +maintainers = [ + { email = "martin.kuehn@dlr.de" } +] +dependencies = [ + "pandas>=1.2.2", + "matplotlib", + "numpy>=1.22,<1.25", + "openpyxl", + "xlrd", + "requests", + "pyxlsb", + "wget", + "folium", + "mapclassify", + "geopandas", + "h5py", + "imageio", + "datetime" +] + +[project.optional-dependencies] +dev = [ + "pyfakefs>=4.6", + "coverage>=7.0.1", + "pylint>=2.13.0,<2.16", + "pylint_json2html==0.4.0" +] + +[project.urls] +Homepage = "https://github.com/SciCompMod/memilio" +Team = "https://memilio.readthedocs.io/en/latest/team.html" + +[tool.setuptools.packages.find] +where = ["."] +include = ["memilio*"] diff --git a/pycode/memilio-plot/setup.py b/pycode/memilio-plot/setup.py deleted file mode 100644 index 9c6a85908a..0000000000 --- a/pycode/memilio-plot/setup.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import subprocess -import sys - -from setuptools import Command, find_packages, setup - -__version__ = '1.0.0' - - -class PylintCommand(Command): - """Custom command to run pylint and get a report as html.""" - description = "Runs pylint and outputs the report as html." - user_options = [] - - def initialize_options(self): - """ """ - from pylint.reporters.json_reporter import JSONReporter - from pylint.reporters.text import ParseableTextReporter, TextReporter - from pylint_json2html import JsonExtendedReporter - - self.lint_modules = ["memilio/"] - self.out_format = "extendedjson" - - self.REPORTERS = { - "parseable": (ParseableTextReporter, "build_pylint/pylint_parseable.txt"), - "text": (TextReporter, "build_pylint/pylint.txt"), - "json": (JSONReporter, "build_pylint/pylint.json"), - "extendedjson": (JsonExtendedReporter, "build_pylint/pylint_extended.json") - } - - def finalize_options(self): - """ """ - self.reporter, self.out_file = self.REPORTERS.get( - self.out_format) # , self.REPORTERS.get("parseable")) - - def run(self): - """ """ - os.makedirs("build_pylint", exist_ok=True) - - # Run pylint - from pylint import lint - with open(self.out_file, "w", encoding="utf-8") as report_file: - options = ["--rcfile=../pylintrc", *self.lint_modules] - - lint.Run(options, reporter=self.reporter( - report_file), do_exit=False) - - -setup( - name='memilio-plot', - version=__version__, - author='DLR-SC', - author_email='martin.kuehn@dlr.de', - maintainer_email='martin.kuehn@dlr.de', - url='https://github.com/SciCompMod/memilio', - description='Part of MEmilio project, plots data to maps or visualizes simulation curves.', - entry_points={ - 'console_scripts': [ - ], - }, - packages=find_packages(where=os.path.dirname(os.path.abspath(__file__))), - long_description='', - test_suite='memilio.plot_test', - install_requires=[ - # smaller pandas versions contain a bug that sometimes prevents reading - # some excel files (e.g. population or mobility data) - 'pandas>=1.2.2', - 'matplotlib', - # smaller numpy versions cause a security issue, 1.25 breaks testing with pyfakefs - 'numpy>=1.22,<1.25', - 'openpyxl', - 'xlrd', - 'requests', - 'pyxlsb', - 'wget', - 'folium', - 'matplotlib', - 'mapclassify', - 'geopandas', - 'h5py', - 'imageio', - 'datetime' - ], - extras_require={ - 'dev': [ - # first support of python 3.11 - 'pyfakefs>=4.6', - 'coverage>=7.0.1', - # pylint 2.16 creates problem with wrapt package version - 'pylint>=2.13.0,<2.16', - 'pylint_json2html==0.4.0', - ], - }, - cmdclass={ - 'pylint': PylintCommand - }, -) diff --git a/pycode/memilio-plot/tools/run_pylint.py b/pycode/memilio-plot/tools/run_pylint.py new file mode 100644 index 0000000000..035473cf6e --- /dev/null +++ b/pycode/memilio-plot/tools/run_pylint.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Generate Pylint reports in jsonextended format for the plot package.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def main() -> int: + project_dir = Path(__file__).resolve().parent.parent + repo_root = project_dir.parent + build_dir = project_dir / "build_pylint" + build_dir.mkdir(exist_ok=True) + output_file = build_dir / "pylint_extended.json" + + cmd = [ + sys.executable, + "-m", + "pylint", + "--rcfile", + str(repo_root / "pylintrc"), + "--load-plugins", + "pylint_json2html", + "--output-format=jsonextended", + "memilio/", + ] + + with output_file.open("w", encoding="utf-8") as stream: + result = subprocess.run( + cmd, + cwd=project_dir, + stdout=stream, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + if result.stderr: + sys.stderr.write(result.stderr) + + if result.returncode: + sys.stderr.write( + "pylint reported issues; see build_pylint/pylint_extended.json for details.\n" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pycode/memilio-simulation/CMakeLists.txt b/pycode/memilio-simulation/CMakeLists.txt index 15c186b68f..2267a59db6 100644 --- a/pycode/memilio-simulation/CMakeLists.txt +++ b/pycode/memilio-simulation/CMakeLists.txt @@ -13,7 +13,16 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") -set(CMAKE_INSTALL_RPATH "${CMAKE_BINARY_DIR}/lib" "${CMAKE_BINARY_DIR}/bin") +set(CMAKE_INSTALL_RPATH + "$ORIGIN" + "$ORIGIN/lib" + "${CMAKE_BINARY_DIR}/lib" + "${CMAKE_BINARY_DIR}/bin" +) + +# Bundled JsonCpp must be linked statically so the wheel stays self-contained. +set(MEMILIO_PREFER_STATIC_JSONCPP ON CACHE BOOL "" FORCE) +set(MEMILIO_PREFER_STATIC_SPDLOG ON CACHE BOOL "" FORCE) if(MEMILIO_USE_BUNDLED_PYBIND11) # Fetch pybind11 @@ -45,7 +54,14 @@ function(add_pymio_module target_name) pybind11_add_module(${target_name} MODULE ${PYBIND11_MODULE_SOURCES}) target_link_libraries(${target_name} PRIVATE ${PYBIND11_MODULE_LINKED_LIBRARIES}) target_include_directories(${target_name} PRIVATE memilio/simulation/bindings) - install(TARGETS ${target_name} LIBRARY DESTINATION memilio/simulation) + set_target_properties(${target_name} PROPERTIES + INSTALL_RPATH "\\$ORIGIN;\\$ORIGIN/lib" + ) + install(TARGETS ${target_name} + LIBRARY DESTINATION . + ARCHIVE DESTINATION . + RUNTIME DESTINATION . + ) endfunction() # build python extensions @@ -100,3 +116,72 @@ add_pymio_module(_simulation_omseirs4 LINKED_LIBRARIES memilio ode_mseirs4 SOURCES memilio/simulation/bindings/models/omseirs4.cpp ) + +set(MEMILIO_SIMULATION_LIBS + memilio + abm + ode_sir + ode_seir + ode_secir + ode_secirvvs + sde_sir + sde_sirs + ode_mseirs4 +) + +foreach(target_name IN LISTS MEMILIO_SIMULATION_LIBS) + if(TARGET ${target_name}) + set_target_properties(${target_name} PROPERTIES + INSTALL_RPATH "\\$ORIGIN" + ) + endif() +endforeach() + +install(TARGETS ${MEMILIO_SIMULATION_LIBS} + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION lib +) + +# If jsoncpp was built as part of the bundled third-party dependencies, make sure its +# shared library is copied into the wheel with the exact SONAME the runtime loader will +# request. Wheels do not preserve symlinks, so install real copies for every relevant +# name. +if(TARGET jsoncpp_lib) + set_target_properties(jsoncpp_lib PROPERTIES + INSTALL_RPATH "\\$ORIGIN" + ) + set(JSONCPP_LIB_FILE $) + install(FILES ${JSONCPP_LIB_FILE} + DESTINATION lib + ) + install(FILES ${JSONCPP_LIB_FILE} + DESTINATION lib + RENAME libjsoncpp.so.26 + ) + install(FILES ${JSONCPP_LIB_FILE} + DESTINATION lib + RENAME libjsoncpp.so + ) +endif() + +if(TARGET spdlog) + get_target_property(_SPDLOG_LIB_TYPE spdlog TYPE) + if(_SPDLOG_LIB_TYPE STREQUAL "SHARED_LIBRARY") + set_target_properties(spdlog PROPERTIES + INSTALL_RPATH "\\$ORIGIN" + ) + set(SPDLOG_LIB_FILE $) + install(FILES ${SPDLOG_LIB_FILE} + DESTINATION lib + ) + install(FILES ${SPDLOG_LIB_FILE} + DESTINATION lib + RENAME libspdlog.so.1.15 + ) + install(FILES ${SPDLOG_LIB_FILE} + DESTINATION lib + RENAME libspdlog.so + ) + endif() +endif() diff --git a/pycode/memilio-simulation/README.md b/pycode/memilio-simulation/README.md index b2de287d6b..1bf8c03fbe 100644 --- a/pycode/memilio-simulation/README.md +++ b/pycode/memilio-simulation/README.md @@ -4,9 +4,9 @@ This package contains Python bindings for the MEmilio C++ library. It enables se ## Installation -Use the provided `setup.py` script to build the bindings and install the package. The script requires CMake and the Scikit-Build packages. Both are installed by the script if not available on the system. The package uses the [Pybind11 C++ library](https://pybind11.readthedocs.io) to create the bindings. +This project is configured via ``pyproject.toml`` and is built with [scikit-build-core](https://scikit-build-core.readthedocs.io). CMake and Ninja must be available on the system. The package uses the [Pybind11 C++ library](https://pybind11.readthedocs.io) to create the bindings. -To install the package, use the command (from the directory containing `setup.py`) +To install the package, use the command (from the directory containing ``pyproject.toml``) ```bash pip install . @@ -16,17 +16,17 @@ This builds the C++ library and C++ Python extension module and copies everythin All the requirements of the [C++ library](../../cpp/README.md) must be met in order to build and use the python bindings. A virtual environment is recommended. -CMake is executed internally by the `setup.py` script. All the options provided by the CMake configuration of the C++ library are available when building the Python extension as well. Additionally, the CMake configuration for the bindings provide the following CMake options: +CMake is executed internally by scikit-build-core. All the options provided by the CMake configuration of the C++ library are available when building the Python extension as well. Additionally, the CMake configuration for the bindings provide the following CMake options: - MEMILIO_USE_BUNDLED_PYBIND11: ON or OFF, default ON. If ON, downloads Pybind11 automatically from a repository during CMake configuration. If OFF, Pybind11 needs to be installed on the system. -When building the bindings, CMake options can be set by appending them to the install command, e.g. +When building the bindings, CMake options can be forwarded with configuration settings, e.g. ```bash -python setup.py install -- -DCMAKE_BUILD_TYPE=Debug -DMEMILIO_USE_BUNDLED_PYBIND11=OFF +pip install . --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug" --config-settings=cmake.args="-DMEMILIO_USE_BUNDLED_PYBIND11=OFF" ``` -Alternatively, the `CMakeCache.txt` in the directory created by Scikit-Build can be edited to set the options. +Alternatively, edit the `CMakeCache.txt` in the directory created by scikit-build-core. ## Development @@ -38,8 +38,6 @@ pip install -e .[dev] This command allows you to work on the code without having to reinstall the package after a change. Note that this only works for changes to Python code. If C++ code is modified, the install command has to be repeated every time. The command also installs all additional dependencies required for development and maintenance. -For development, it may be easier to use the alternative command `python setup.py ` which provides better configuration and observation of the C++ build process. - The [bindings](memilio/simulation/bindings/) folder contains all the C++ code to expose the MEmilio library to Python and follows its structure, expect for the models that are bundled in a subfolder. In order to add a new model, the following steps need to be taken: diff --git a/pycode/memilio-simulation/pyproject.toml b/pycode/memilio-simulation/pyproject.toml new file mode 100644 index 0000000000..3a7d07285e --- /dev/null +++ b/pycode/memilio-simulation/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = [ + "scikit-build-core>=0.9.0", + "setuptools>=68", + "wheel" +] +build-backend = "scikit_build_core.build" + +[project] +name = "memilio-simulation" +version = "1.0.0" +description = "Part of MEmilio project, python bindings to the C++ libraries that contain the models and simulations." +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "MEmilio Team" }] +maintainers = [ + { email = "martin.kuehn@dlr.de" } +] +dependencies = [] + +[project.optional-dependencies] +dev = [ + "numpy>=1.22,<1.25", + "pandas>=2.0.0" +] + +[project.urls] +Homepage = "https://github.com/SciCompMod/memilio" +Team = "https://memilio.readthedocs.io/en/latest/team.html" + +[tool.scikit-build] +cmake.version = ">=3.13" +cmake.args = ["-DMEMILIO_BUILD_SHARED_LIBS:BOOL=ON"] +wheel.packages = ["memilio"] +wheel.install-dir = "memilio/simulation" diff --git a/pycode/memilio-simulation/setup.py b/pycode/memilio-simulation/setup.py deleted file mode 100644 index 037a93df8e..0000000000 --- a/pycode/memilio-simulation/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import subprocess -import sys - -from setuptools import find_packages, setup - -try: - from skbuild import setup -except ImportError: - print('scikit-build is required to build from source.') - print('Installation: python -m pip install scikit-build') - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "scikit-build"]) - from skbuild import setup - -__version__ = '1.0.0' - -setup( - name='memilio-simulation', version=__version__, author='DLR-SC', - author_email='daniel.abele@dlr.de', maintainer_email='Martin.Kuehn@DLR.de', - url='https://github.com/SciCompMod/memilio', - description='Part of MEmilio project, python bindings to the C++ libraries that contain the models and simulations.', - packages=find_packages(where=os.path.dirname(os.path.abspath(__file__))), - # need shared libs so there is one shared log level - cmake_args=['-DMEMILIO_BUILD_SHARED_LIBS:BOOL=ON'], - install_requires=[ - ], - extras_require={ - 'dev': [ - # smaller numpy versions cause a security issue, 1.25 breaks testing with pyfakefs - 'numpy>=1.22,<1.25', - # smaller pandas versions contain a bug that sometimes prevents reading - # some excel files (e.g. population or mobility data) - 'pandas>=2.0.0', - ], - }, - long_description='', test_suite='memilio.simulation_test', -) diff --git a/pycode/memilio-simulation/tools/generate_stubs.py b/pycode/memilio-simulation/tools/generate_stubs.py index 6ac97d1d88..c22782a08c 100644 --- a/pycode/memilio-simulation/tools/generate_stubs.py +++ b/pycode/memilio-simulation/tools/generate_stubs.py @@ -17,23 +17,29 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################# +import importlib.util import os +import shutil import subprocess import sys -import importlib.util -import shutil -setup_content_protected_module = f""" -from setuptools import setup, find_packages - -setup( - name='memilio-stubs', - version='0.1', - packages=['memilio-stubs'], - package_data={{ - 'memilio-stubs': ['simulation/*.pyi'], - }}, -) +pyproject_content_protected_module = """[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "memilio-stubs" +version = "0.1" +description = "Stubs for the memilio.simulation package." +requires-python = ">=3.8" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["."] +include = ["memilio-stubs"] + +[tool.setuptools.package-data] +"memilio-stubs" = ["simulation/*.pyi"] """ # Define if the generated stubs of mypy should be configured @@ -71,7 +77,8 @@ # memilio-stubs/simulation module needs same structure as memilio/simulation # test --include-docstrings, --doc-dir PATH for better docs subprocess.check_call( - ['stubgen', '--include-docstrings', '-o', package_dir, '-p', 'memilio.simulation']) + ['stubgen', '--include-docstrings', '-o', package_dir, '-p', + 'memilio.simulation']) # TODO: # - fix numpy.float64[m, 1] to @@ -163,8 +170,8 @@ shutil.move(os.path.join(package_dir, "memilio"), os.path.join(package_dir, "memilio-stubs")) - # create setup.py and install package - with open(os.path.join(package_dir, "setup.py"), "w") as setup_file: - setup_file.write(setup_content_protected_module) + # create pyproject.toml and install package + with open(os.path.join(package_dir, "pyproject.toml"), "w", encoding="utf-8") as pyproject_file: + pyproject_file.write(pyproject_content_protected_module) subprocess.check_call( [python_interpreter, '-m', 'pip', 'install', package_dir]) diff --git a/pycode/memilio-surrogatemodel/README.md b/pycode/memilio-surrogatemodel/README.md index c1ac0ef10e..951469e853 100644 --- a/pycode/memilio-surrogatemodel/README.md +++ b/pycode/memilio-surrogatemodel/README.md @@ -4,8 +4,8 @@ This package contains machine learning based surrogate models that make predicti ## Installation -Use the provided `setup.py` script install the package. -To install the package, use the command (from the directory containing `setup.py`) +Use the provided `pyproject.toml` file to install the package. +To install the package, use the command (from the directory containing `pyproject.toml`) ```bash pip install . diff --git a/pycode/memilio-surrogatemodel/pyproject.toml b/pycode/memilio-surrogatemodel/pyproject.toml new file mode 100644 index 0000000000..706f45993d --- /dev/null +++ b/pycode/memilio-surrogatemodel/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "memilio-surrogatemodel" +version = "1.0.0" +description = "Part of MEmilio project, implementation of surrogate models for the existing models in MEmilio." +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "MEmilio Team" }] +maintainers = [ + { email = "martin.kuehn@dlr.de" } +] +dependencies = [ + "pandas>=1.2.2", + "progress", + "numpy>=1.22,<1.25", + "tensorflow", + "matplotlib", + "scikit-learn" +] + +[project.optional-dependencies] +dev = [ + "pyfakefs>=4.6", + "coverage>=7.0.1" +] + +[project.urls] +Homepage = "https://github.com/SciCompMod/memilio" +Team = "https://memilio.readthedocs.io/en/latest/team.html" + +[tool.setuptools.packages.find] +where = ["."] +include = ["memilio*"] diff --git a/pycode/memilio-surrogatemodel/setup.py b/pycode/memilio-surrogatemodel/setup.py deleted file mode 100644 index cbd176f47e..0000000000 --- a/pycode/memilio-surrogatemodel/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -from setuptools import find_packages, setup - -__version__ = '1.0.0' - -setup( - name='memilio-surrogatemodel', version=__version__, author='DLR-SC', - author_email='henrik.zunker@dlr.de', - maintainer_email='martin.kuehn@dlr.de', - url='https://github.com/SciCompMod/memilio', - description='Part of MEmilio project, implementation of surrogate models for the existing models in MEmilio.', - packages=find_packages( - where=os.path.dirname(os.path.abspath(__file__))), - install_requires=[ - # smaller pandas versions contain a bug that sometimes prevents reading - # some excel files (e.g. population or mobility data) - 'pandas>=1.2.2', - 'progress', - # smaller numpy versions cause a security issue, 1.25 breaks testing with pyfakefs - 'numpy>=1.22,<1.25', - 'tensorflow', - 'matplotlib', - 'scikit-learn', ], - extras_require={'dev': [ - # first support of python 3.11 - 'pyfakefs>=4.6', - 'coverage>=7.0.1', - ], }, - long_description='', test_suite='memilio.surrogatemodel_test',)