WIP: added source_copy: False option

Author: Maria Francine Therese Ruiz Lapid

lib/pavilion/builder.py

@@ -26,6 +26,8 @@
 from pavilion.test_config import parse_timeout
 from pavilion.test_config.spack import SpackEnvConfig
 
+from pavilion.output import dbg_print
+
 class TestBuilder:
     """Manages a test build and their organization.
 
@@ -638,7 +640,6 @@ def _setup_build_dir(self, dest, tracker: BuildTracker):
         :param tracker: Build tracker for this build.
         :return: None
         """
-
         umask = os.umask(0)
         os.umask(umask)
 
@@ -666,18 +667,23 @@ def _setup_build_dir(self, dest, tracker: BuildTracker):
 
         elif src_path.is_dir():
             # Recursively copy the src directory to the build directory.
+            # FRANCINE: add option to symlink everything recursively rather than copy
             tracker.update(
                 state=STATES.BUILDING,
                 note=("Copying source directory {} for build {} "
                       "as the build directory."
                       .format(src_path, dest)))
 
-            utils.copytree(
-                src_path.as_posix(),
-                dest.as_posix(),
-                copy_function=shutil.copyfile,
-                copystat=utils.make_umask_filtered_copystat(umask),
-                symlinks=True)
+            source_copy = self._config.get('source_copy')
+            if source_copy.lower() == 'true':
+                utils.copytree(
+                    src_path.as_posix(),
+                    dest.as_posix(),
+                    copy_function=shutil.copyfile,
+                    copystat=utils.make_umask_filtered_copystat(umask),
+                    symlinks=True)
+            else:
+                utils.symlinktree(src_path, dest)
 
         elif src_path.is_file():
             category, subtype = utils.get_mime_type(src_path)

lib/pavilion/test_config/file_format.py

@@ -663,6 +663,10 @@ class TestConfigLoader(yc.YamlConfigLoader):
                               "source, tracking changes by "
                               "file size/timestamp/hash."
                 ),
+                yc.StrElem(
+                    'source_copy', default='True', choices=['true', 'false', 'True', 'False'],
+                    help_text="Whether to copy everything in source_path into the working dir."
+                ),
                 yc.KeyedElem(
                     'spack', elements=[
                         yc.ListElem(

lib/pavilion/utils.py

@@ -14,6 +14,8 @@
 from typing import Iterator, Union, TextIO
 from typing import List, Dict
 
+from pavilion.output import dbg_print
+
 
 def glob_to_re(glob):
     """Translate the given glob to one that is compatible with (extended) grep.
@@ -193,6 +195,18 @@ def path_is_external(path: Path):
     return not_up_refs - up_refs <= 0
 
 
+def symlinktree(source_directory, destination_directory):
+    for root, dirs, files in os.walk(source_directory):
+        for file in files:
+            src_path = os.path.join(root, file)
+            rel_path = os.path.relpath(src_path, source_directory)
+            dst_path = os.path.join(destination_directory, rel_path)
+
+            # Create
+            os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+            os.symlink(src_path, dst_path)
+
+
 def flat_walk(path, *args, **kwargs) -> Iterator[Path]:
     """Perform an os.walk on path, but simply generate each item walked over.