|
| 1 | +# Copyright Swiss National Supercomputing Centre (CSCS/ETH Zurich) |
| 2 | +# ReFrame Project Developers. See the top-level LICENSE file for details. |
| 3 | +# |
| 4 | +# SPDX-License-Identifier: BSD-3-Clause |
| 5 | + |
| 6 | +import reframe as rfm |
| 7 | +import reframe.utility.sanity as sn |
| 8 | +import uenv |
| 9 | +import os |
| 10 | +import reframe.utility.osext as osext |
| 11 | + |
| 12 | + |
| 13 | +qe_references = { |
| 14 | + 'Au surf': { |
| 15 | + 'gh200': {'time_run': (50, None, 0.05, 's')}, |
| 16 | + 'mi300': {'time_run': (112, None, 0.05, 's')}, |
| 17 | + 'mi200': {'time_run': (78, None, 0.05, 's')}, |
| 18 | + }, |
| 19 | +} |
| 20 | + |
| 21 | + |
| 22 | +slurm_config = { |
| 23 | + 'Au surf': { |
| 24 | + 'gh200': { |
| 25 | + 'nodes': 1, |
| 26 | + 'ntasks-per-node': 4, |
| 27 | + 'cpus-per-task': 72, |
| 28 | + 'walltime': '0d0h20m0s', |
| 29 | + 'extra_job_options': [], |
| 30 | + 'gpu': True, |
| 31 | + }, |
| 32 | + 'mi200': { |
| 33 | + 'nodes': 1, |
| 34 | + 'ntasks-per-node': 8, |
| 35 | + 'cpus-per-task': 8, |
| 36 | + 'walltime': '0d0h20m0s', |
| 37 | + 'extra_job_options': [], |
| 38 | + 'gpu': True, |
| 39 | + }, |
| 40 | + 'mi300': { |
| 41 | + 'nodes': 1, |
| 42 | + 'ntasks-per-node': 12, |
| 43 | + 'gpus-per-task': 1, |
| 44 | + 'cpus-per-task': 8, |
| 45 | + 'extra_job_options': ['--constraint=amdgpu_tpx'], |
| 46 | + 'walltime': '0d0h20m0s', |
| 47 | + 'gpu': True, |
| 48 | + }, |
| 49 | + 'zen2': { |
| 50 | + 'nodes': 1, |
| 51 | + 'ntasks-per-node': 16, |
| 52 | + 'cpus-per-task': 8, |
| 53 | + 'walltime': '0d0h20m0s', |
| 54 | + 'extra_job_options': [], |
| 55 | + 'gpu': False, |
| 56 | + }, |
| 57 | + }, |
| 58 | +} |
| 59 | + |
| 60 | + |
| 61 | +def uenv_uarch(): |
| 62 | + uenv_name = os.environ.get("UENV", None) |
| 63 | + if not uenv_name: |
| 64 | + return uenv_name |
| 65 | + uenv_inspect_cmd = f"uenv image inspect --format={{uarch}} {uenv_name}" |
| 66 | + return osext.run_command(uenv_inspect_cmd, shell=True).stdout.strip() |
| 67 | + |
| 68 | + |
| 69 | +class QeSiriusCheckUENV(rfm.RunOnlyRegressionTest): |
| 70 | + pwx_executable = 'pw.x' |
| 71 | + maintainers = ['simonpintarelli', 'SSA'] |
| 72 | + valid_systems = ['+uenv +amdgpu', '+uenv +nvgpu'] |
| 73 | + |
| 74 | + @run_after('setup') |
| 75 | + def skip_unsupported_uenv(self): |
| 76 | + _uarch = uenv.uarch(self.current_partition) |
| 77 | + self.skip_if(_uarch != uenv_uarch(), |
| 78 | + f'this uenv does not support {_uarch}') |
| 79 | + |
| 80 | + @run_before('run') |
| 81 | + def prepare_run(self): |
| 82 | + self.uarch = uenv.uarch(self.current_partition) |
| 83 | + config = slurm_config[self.test_name][self.uarch] |
| 84 | + # sbatch options |
| 85 | + self.job.options = [ |
| 86 | + f'--nodes={config["nodes"]}', |
| 87 | + ] |
| 88 | + self.job.options += config['extra_job_options'] |
| 89 | + |
| 90 | + self.num_tasks_per_node = config['ntasks-per-node'] |
| 91 | + self.num_tasks = config['nodes'] * self.num_tasks_per_node |
| 92 | + self.num_cpus_per_task = config['cpus-per-task'] |
| 93 | + self.ntasks_per_core = 1 |
| 94 | + self.time_limit = config['walltime'] |
| 95 | + |
| 96 | + # environment variables |
| 97 | + self.env_vars['OMP_NUM_THREADS'] = config['cpus-per-task'] |
| 98 | + self.env_vars['SIRIUS_PRINT_TIMING'] = '5' |
| 99 | + self.env_vars['SIRIUS_PRINT_MPI_LAYOUT'] = '1' |
| 100 | + self.env_vars['SIRIUS_VERBOSITY'] = '2' |
| 101 | + self.env_vars['SLURM_HINT'] = 'nomultithread' |
| 102 | + if self.uarch == 'gh200': |
| 103 | + self.env_vars['MPICH_GPU_SUPPORT_ENABLED'] = '1' |
| 104 | + self.env_vars['OMP_NUM_THREADS'] = str(20) |
| 105 | + if self.uarch == 'mi200': |
| 106 | + self.env_vars['MPICH_GPU_SUPPORT_ENABLED'] = '1' |
| 107 | + self.env_vars['OMP_NUM_THREADS'] = str(8) |
| 108 | + if self.uarch == 'mi300': |
| 109 | + self.env_vars['MPICH_GPU_SUPPORT_ENABLED'] = '1' |
| 110 | + self.env_vars['OMP_NUM_THREADS'] = str(8) |
| 111 | + if self.uarch in ('mi300', 'mi200'): |
| 112 | + self.env_vars['PIKA_MPI_ENABLE_POOL'] = '1' |
| 113 | + self.env_vars['PIKA_MPI_COMPLETION_MODE'] = '28' |
| 114 | + self.env_vars['DLAF_BAND_TO_TRIDIAG_1D_BLOCK_SIZE_BASE'] = '2048' |
| 115 | + self.env_vars['DLAF_NUM_NP_GPU_STREAMS'] = '4' |
| 116 | + self.env_vars['DLAF_NUM_HP_GPU_STREAMS'] = '4' |
| 117 | + |
| 118 | + # set reference |
| 119 | + if self.uarch is not None and self.uarch in qe_references[self.test_name]: |
| 120 | + self.reference = { |
| 121 | + self.current_partition.fullname: |
| 122 | + qe_references[self.test_name][self.uarch] |
| 123 | + } |
| 124 | + |
| 125 | + @sanity_function |
| 126 | + def assert_energy_diff(self): |
| 127 | + energy = sn.extractsingle( |
| 128 | + r'^!\s+total energy\s+=\s+(?P<energy>\S+)', |
| 129 | + self.stdout, |
| 130 | + 'energy', |
| 131 | + float, |
| 132 | + item=-1, |
| 133 | + ) |
| 134 | + energy_diff = sn.abs(energy - self.energy_reference) |
| 135 | + successful_termination = sn.assert_found(r'JOB DONE', self.stdout) |
| 136 | + correct_energy = sn.assert_lt(energy_diff, 5e-4) |
| 137 | + return sn.all( |
| 138 | + [ |
| 139 | + successful_termination, |
| 140 | + correct_energy, |
| 141 | + ] |
| 142 | + ) |
| 143 | + |
| 144 | + # INFO: The name of this function needs to match with the reference dict! |
| 145 | + @performance_function('s') |
| 146 | + def time_run(self): |
| 147 | + return sn.extractsingle( |
| 148 | + r'electrons.+\s(?P<wtime>\S+)s WALL', self.stdout, 'wtime', float |
| 149 | + ) |
| 150 | + |
| 151 | + |
| 152 | +class QeSiriusCheckAuSurfUENV(QeSiriusCheckUENV): |
| 153 | + test_name = 'Au surf' |
| 154 | + executable_opts = ['-i', 'ausurf.in'] |
| 155 | + energy_reference = -11427.09017218 |
| 156 | + |
| 157 | + |
| 158 | +@rfm.simple_test |
| 159 | +class QeCheckAuSurfUENVExec(QeSiriusCheckAuSurfUENV): |
| 160 | + valid_prog_environs = ['+uenv +q-e-sirius'] |
| 161 | + tags = {'uenv', 'production', 'bencher'} |
| 162 | + |
| 163 | + @run_after('setup') |
| 164 | + def setup_executable(self): |
| 165 | + self.executable = f'pw.x' |
| 166 | + uarch = uenv.uarch(self.current_partition) |
| 167 | + if uarch == 'gh200': |
| 168 | + self.executable = f'./mps-wrapper.sh pw.x' |
| 169 | + if uarch == 'mi200': |
| 170 | + self.executable = \ |
| 171 | + f'./amdgpu-wrapper.sh pw.x -sirius_cfg sirius-amd.json' |
| 172 | + if uarch == 'mi300': |
| 173 | + self.executable = \ |
| 174 | + f'./tpx-wrapper.sh pw.x -sirius_cfg sirius-amd.json' |
0 commit comments