Add an explicit setup_py metadata to indicate the need to build

Author: mdboom

doc/custom_benchmarks.rst

@@ -333,6 +333,7 @@ tool.extra_opts [str]     X
 tool.inherits   file           
 tool.runscript  file      X    
 tool.datadir    file      X    
+tool.setup_py   bool
 =============== ===== === === ===
 
 "R": required
@@ -342,3 +343,5 @@ tool.datadir    file      X
 * tags: optional list of names to group benchmarks
 * extra_opts: optional list of args to pass to ``tool.runscript``
 * runscript: the benchmark script to use instead of run_benchmark.py.
+* setup_py: when ``true``, run the setup.py in the benchmark directory to build
+  and install artifacts, such as C extensions

pyperformance/_benchmark.py

@@ -164,6 +164,10 @@ def runscript(self):
     def extra_opts(self):
         return self._get_metadata_value('extra_opts', ())
 
+    @property
+    def setup_py(self):
+        return self._get_metadata_value('setup_py', False)
+
     # Other metadata keys:
     # * base
     # * python

pyperformance/_benchmark_metadata.py

@@ -32,6 +32,7 @@
     'datadir': None,
     'runscript': None,
     'extra_opts': None,
+    'setup_py': None,
 }
 
 
@@ -228,6 +229,9 @@ def _resolve_value(field, value, rootdir):
         for opt in value:
             if not opt or not isinstance(opt, str):
                 raise TypeError(f'extra_opts should be a list of strings, got {value!r}')
+    elif field == 'setup_py':
+        if not isinstance(value, bool):
+            raise TypeError(f'setup_py should be a bool, got {value!r}')
     else:
         raise NotImplementedError(field)
     return value

pyperformance/data-files/benchmarks/bm_ctypes/pyproject.toml

@@ -8,3 +8,4 @@ dynamic = ["version"]
 [tool.pyperformance]
 name = "ctypes"
 tags = "extension"
+setup_py = true

pyperformance/venv.py

@@ -24,6 +24,8 @@ def from_benchmarks(cls, benchmarks):
         for bench in benchmarks or ():
             filename = bench.requirements_lockfile
             self._add_from_file(filename)
+            if bench.setup_py:
+                self._add(os.path.dirname(bench.metafile))
         return self
 
     def __init__(self):
@@ -241,10 +243,6 @@ def ensure_reqs(self, requirements=None):
             bench = requirements
             requirements = Requirements.from_benchmarks([bench])
 
-            benchmark_dir = os.path.dirname(bench.metafile)
-            if os.path.isfile(os.path.join(benchmark_dir, "setup.py")):
-                requirements._add(benchmark_dir)
-
         # Every benchmark must depend on pyperf.
         if bench is not None and not requirements.get('pyperf'):
             basereqs = Requirements.from_file(REQUIREMENTS_FILE)