Skip to content

Commit 453adfd

Browse files
chr-hertelchr-hertel
committed
feature #815 Level up runner for convenience when running large sets of tests (chr-hertel)
This PR was squashed before being merged into the main branch. Discussion ---------- Level up runner for convenience when running large sets of tests | Q | A | ------------- | --- | Bug fix? | no | New feature? | yes | Docs? | no | Issues | | License | MIT <img width="1496" height="985" alt="image" src="https://github.com/user-attachments/assets/d26c05ab-a999-42eb-80d1-876ffee0f878" /> <img width="1516" height="1042" alt="image" src="https://github.com/user-attachments/assets/e4951709-cea9-4db0-a824-63004da3b609" /> Commits ------- 15e2223 Level up runner for convenience when running large sets of tests
2 parents 73beb39 + 15e2223 commit 453adfd

File tree

1 file changed

+136
-45
lines changed

1 file changed

+136
-45
lines changed

‎examples/runner‎

Lines changed: 136 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use Symfony\Component\Console\Input\InputArgument;
77
use Symfony\Component\Console\Input\InputInterface;
88
use Symfony\Component\Console\Input\InputOption;
99
use Symfony\Component\Console\Output\OutputInterface;
10+
use Symfony\Component\Console\Question\ChoiceQuestion;
1011
use Symfony\Component\Console\SingleCommandApplication;
1112
use Symfony\Component\Console\Style\SymfonyStyle;
1213
use Symfony\Component\Finder\Finder;
@@ -22,6 +23,7 @@ $app = (new SingleCommandApplication('Symfony AI Example Runner'))
2223
->setDescription('Runs all Symfony AI examples in folder examples/')
2324
->addArgument('subdirectories', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'List of subdirectories to run examples from, e.g. "anthropic" or "huggingface".')
2425
->addOption('filter', 'f', InputOption::VALUE_REQUIRED, 'Filter examples by name, e.g. "audio" or "toolcall".')
26+
->addOption('chunk', 'c', InputOption::VALUE_REQUIRED, 'Number of examples to run in parallel per chunk.', 30)
2527
->setCode(function (InputInterface $input, OutputInterface $output) {
2628
$io = new SymfonyStyle($input, $output);
2729
$io->title('Symfony AI Examples');
@@ -54,60 +56,100 @@ $app = (new SingleCommandApplication('Symfony AI Example Runner'))
5456
->notName(['bootstrap.php', '_[a-z\-]*.php'])
5557
->files();
5658

57-
$io->comment(sprintf('Found %d example(s) to run.', count($examples)));
59+
$chunkSize = (int) $input->getOption('chunk');
60+
$examplesArray = iterator_to_array($examples);
61+
$chunks = array_chunk($examplesArray, $chunkSize);
62+
63+
$io->comment(sprintf('Found %d example(s) to run in %d chunk(s) of max %d examples.', count($examplesArray), count($chunks), $chunkSize));
5864

5965
/** @var array{example: SplFileInfo, process: Process} $exampleRuns */
6066
$exampleRuns = [];
61-
foreach ($examples as $example) {
62-
$exampleRuns[] = [
63-
'example' => $example,
64-
'process' => $process = new Process(['php', $example->getRealPath()]),
65-
];
66-
$process->start();
67-
}
6867

69-
$section = $output->section();
70-
$renderTable = function () use ($exampleRuns, $section) {
71-
$section->clear();
72-
$table = new Table($section);
73-
$table->setHeaders(['Example', 'State', 'Output']);
74-
foreach ($exampleRuns as $run) {
75-
/** @var SplFileInfo $example */
76-
/** @var Process $process */
77-
['example' => $example, 'process' => $process] = $run;
78-
79-
$output = str_replace(PHP_EOL, ' ', $process->getOutput());
80-
$output = strlen($output) <= 100 ? $output : substr($output, 0, 100).'...';
81-
$emptyOutput = 0 === strlen(trim($output));
82-
83-
$state = 'Running';
84-
if ($process->isTerminated()) {
85-
$success = $process->isSuccessful() && !$emptyOutput;
86-
$state = $success ? '<info>Finished</info>'
87-
: (1 === $run['process']->getExitCode() || $emptyOutput ? '<error>Failed</error>' : '<comment>Skipped</comment>');
68+
foreach ($chunks as $chunkIndex => $chunk) {
69+
$io->section(sprintf('Running chunk %d/%d (%d examples)', $chunkIndex + 1, count($chunks), count($chunk)));
70+
71+
$chunkRuns = [];
72+
foreach ($chunk as $example) {
73+
$run = [
74+
'example' => $example,
75+
'process' => $process = new Process(['php', $example->getRealPath()]),
76+
];
77+
$chunkRuns[] = $run;
78+
$exampleRuns[] = $run;
79+
$process->start();
80+
}
81+
82+
$section = $output->section();
83+
$renderTable = function () use ($chunkRuns, $section) {
84+
$section->clear();
85+
$table = new Table($section);
86+
$table->setHeaders(['Example', 'State', 'Output']);
87+
foreach ($chunkRuns as $run) {
88+
/** @var SplFileInfo $example */
89+
/** @var Process $process */
90+
['example' => $example, 'process' => $process] = $run;
91+
92+
$output = str_replace(PHP_EOL, ' ', $process->getOutput());
93+
$output = strlen($output) <= 100 ? $output : substr($output, 0, 100).'...';
94+
$emptyOutput = 0 === strlen(trim($output));
95+
96+
$state = 'Running';
97+
if ($process->isTerminated()) {
98+
$success = $process->isSuccessful() && !$emptyOutput;
99+
$state = $success ? '<info>Finished</info>'
100+
: (1 === $run['process']->getExitCode() || $emptyOutput ? '<error>Failed</error>' : '<comment>Skipped</comment>');
101+
}
102+
103+
$table->addRow([$example->getRelativePathname(), $state, $output]);
88104
}
105+
$table->render();
106+
};
89107

90-
$table->addRow([$example->getRelativePathname(), $state, $output]);
108+
$chunkRunning = fn () => array_reduce($chunkRuns, fn ($running, $example) => $running || $example['process']->isRunning(), false);
109+
while ($chunkRunning()) {
110+
$renderTable();
111+
sleep(1);
91112
}
92-
$table->render();
93-
};
94113

95-
$examplesRunning = fn () => array_reduce($exampleRuns, fn ($running, $example) => $running || $example['process']->isRunning(), false);
96-
while ($examplesRunning()) {
97114
$renderTable();
98-
sleep(1);
115+
$io->newLine();
99116
}
100117

101-
$renderTable();
102-
$io->newLine();
118+
// Group results by directory
119+
$resultsByDirectory = [];
120+
foreach ($exampleRuns as $run) {
121+
$directory = trim(str_replace(__DIR__, '', $run['example']->getPath()), '/');
122+
if (!isset($resultsByDirectory[$directory])) {
123+
$resultsByDirectory[$directory] = ['successful' => 0, 'skipped' => 0, 'failed' => 0];
124+
}
103125

104-
$successCount = array_reduce($exampleRuns, function ($count, $example) {
105-
if ($example['process']->isSuccessful() && strlen(trim($example['process']->getOutput())) > 0) {
106-
return $count + 1;
126+
$emptyOutput = 0 === strlen(trim($run['process']->getOutput()));
127+
if ($run['process']->isSuccessful() && !$emptyOutput) {
128+
$resultsByDirectory[$directory]['successful']++;
129+
} elseif (1 === $run['process']->getExitCode() || $emptyOutput) {
130+
$resultsByDirectory[$directory]['failed']++;
131+
} else {
132+
$resultsByDirectory[$directory]['skipped']++;
107133
}
108-
return $count;
109-
}, 0);
134+
}
135+
136+
ksort($resultsByDirectory);
137+
138+
$io->section('Results by Directory');
139+
$resultsTable = new Table($output);
140+
$resultsTable->setHeaders(['Directory', 'Successful', 'Skipped', 'Failed']);
141+
foreach ($resultsByDirectory as $directory => $stats) {
142+
$resultsTable->addRow([
143+
$directory ?: '.',
144+
sprintf('<info>%d</info>', $stats['successful']),
145+
sprintf('<comment>%d</comment>', $stats['skipped']),
146+
sprintf('<error>%d</error>', $stats['failed']),
147+
]);
148+
}
149+
$resultsTable->render();
150+
$io->newLine();
110151

152+
$successCount = array_sum(array_column($resultsByDirectory, 'successful'));
111153
$totalCount = count($exampleRuns);
112154

113155
if ($successCount < $totalCount) {
@@ -116,11 +158,60 @@ $app = (new SingleCommandApplication('Symfony AI Example Runner'))
116158
$io->success(sprintf('All %d examples ran successfully!', $totalCount));
117159
}
118160

119-
foreach ($exampleRuns as $run) {
120-
if (!$run['process']->isSuccessful()) {
121-
$io->section('Error in ' . $run['example']->getRelativePathname());
122-
$output = $run['process']->getErrorOutput();
123-
$io->text('' !== $output ? $output : $run['process']->getOutput());
161+
if ($output->isVerbose()) {
162+
foreach ($exampleRuns as $run) {
163+
if (!$run['process']->isSuccessful()) {
164+
$io->section('Error in ' . $run['example']->getRelativePathname());
165+
$output = $run['process']->getErrorOutput();
166+
$io->text('' !== $output ? $output : $run['process']->getOutput());
167+
}
168+
}
169+
}
170+
171+
// Interactive retry for failed examples
172+
if ($input->isInteractive()) {
173+
$failedRuns = array_filter($exampleRuns, fn ($run) => !$run['process']->isSuccessful());
174+
175+
while (count($failedRuns) > 0) {
176+
$io->newLine();
177+
$choices = [];
178+
$choiceMap = [];
179+
foreach ($failedRuns as $key => $run) {
180+
$choice = $run['example']->getRelativePathname();
181+
$choices[] = $choice;
182+
$choiceMap[$choice] = $key;
183+
}
184+
$choices[] = 'Exit';
185+
186+
$question = new ChoiceQuestion(
187+
sprintf('Select a failed example to re-run (%d remaining)', count($failedRuns)),
188+
$choices,
189+
count($choices) - 1
190+
);
191+
$question->setErrorMessage('Choice %s is invalid.');
192+
193+
$selected = $io->askQuestion($question);
194+
195+
if ('Exit' === $selected) {
196+
break;
197+
}
198+
199+
$runKey = $choiceMap[$selected];
200+
$run = $failedRuns[$runKey];
201+
202+
$io->section(sprintf('Re-running: %s', $run['example']->getRelativePathname()));
203+
$process = new Process(['php', $run['example']->getRealPath()]);
204+
$process->run(function ($type, $buffer) use ($output) {
205+
$output->write($buffer);
206+
});
207+
208+
if ($process->isSuccessful()) {
209+
unset($failedRuns[$runKey]);
210+
}
211+
}
212+
213+
if ($successCount !== $totalCount && count($failedRuns) === 0) {
214+
$io->success('All previously failed examples now pass!');
124215
}
125216
}
126217

0 commit comments

Comments
 (0)