From 0f6fba35d4655903607341bf226e36658933d335 Mon Sep 17 00:00:00 2001 From: Mark Shipman Date: Sun, 26 Oct 2025 20:58:16 +0000 Subject: [PATCH 1/4] mipro_optimizer_v2 --- dspy/teleprompt/mipro_optimizer_v2.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py index cf76f7daf9..1e39065f1b 100644 --- a/dspy/teleprompt/mipro_optimizer_v2.py +++ b/dspy/teleprompt/mipro_optimizer_v2.py @@ -20,6 +20,9 @@ set_signature, ) +from optuna.samplers import BaseSampler +from mpromptune import qEI + if TYPE_CHECKING: import optuna @@ -114,6 +117,8 @@ def compile( fewshot_aware_proposer: bool = True, requires_permission_to_run: bool | None = None, # deprecated provide_traceback: bool | None = None, + sampler: BaseSampler | None = None, + n_jobs: int = 1 ) -> Any: if requires_permission_to_run == False: logger.warning( @@ -214,6 +219,8 @@ def compile( minibatch_size, minibatch_full_eval_steps, seed, + sampler, + n_jobs ) return best_program @@ -438,6 +445,8 @@ def _optimize_prompt_parameters( minibatch_size: int, minibatch_full_eval_steps: int, seed: int, + sampler: BaseSampler, + n_jobs: int ) -> Any | None: import optuna @@ -577,8 +586,15 @@ def objective(trial): return score - sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) - study = optuna.create_study(direction="maximize", sampler=sampler) + if sampler: + _sampler = sampler + if type(_sampler) == qEI.Sampler: + _sampler.create_search_space(instruction_candidates, demo_candidates) + else: + # backwards compaitable + _sampler= optuna.samplers.TPESampler(seed=seed, multivariate=True) + + study = optuna.create_study(direction="maximize", sampler=_sampler) default_params = {f"{i}_predictor_instruction": 0 for i in range(len(program.predictors()))} if demo_candidates: @@ -591,7 +607,8 @@ def objective(trial): value=default_score, ) study.add_trial(trial) - study.optimize(objective, n_trials=num_trials) + study.optimize(objective, n_trials=n_trials, n_jobs=n_jobs) + self.study = study # Attach logs to best program if best_program is not None and self.track_stats: @@ -700,7 +717,8 @@ def _select_and_insert_instructions_and_demos( predictor.demos = demo_candidates[i][demos_idx] trial_logs[trial_num][f"{i}_predictor_demos"] = demos_idx chosen_params.append(f"Predictor {i}: Few-Shot Set {demos_idx}") - raw_chosen_params[f"{i}_predictor_demos"] = instruction_idx + raw_chosen_params[f"{i}_predictor_demos"] = demos_idx +# raw_chosen_params[f"{i}_predictor_demos"] = instruction_idx return chosen_params, raw_chosen_params From 3fd30255bf8ae214b9bb76b7b2cd82986662c160 Mon Sep 17 00:00:00 2001 From: Mark Shipman Date: Mon, 27 Oct 2025 15:07:24 +0000 Subject: [PATCH 2/4] passing setup_config instead of sampler --- dspy/teleprompt/mipro_optimizer_v2.py | 33 +++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py index 1e39065f1b..9c380e75c2 100644 --- a/dspy/teleprompt/mipro_optimizer_v2.py +++ b/dspy/teleprompt/mipro_optimizer_v2.py @@ -2,9 +2,7 @@ import random from collections import defaultdict from typing import TYPE_CHECKING, Any, Callable, Literal - import numpy as np - import dspy from dspy.evaluate.evaluate import Evaluate from dspy.propose import GroundedProposer @@ -20,7 +18,6 @@ set_signature, ) -from optuna.samplers import BaseSampler from mpromptune import qEI if TYPE_CHECKING: @@ -117,8 +114,7 @@ def compile( fewshot_aware_proposer: bool = True, requires_permission_to_run: bool | None = None, # deprecated provide_traceback: bool | None = None, - sampler: BaseSampler | None = None, - n_jobs: int = 1 + **kwargs ) -> Any: if requires_permission_to_run == False: logger.warning( @@ -219,8 +215,7 @@ def compile( minibatch_size, minibatch_full_eval_steps, seed, - sampler, - n_jobs + kwargs ) return best_program @@ -445,8 +440,7 @@ def _optimize_prompt_parameters( minibatch_size: int, minibatch_full_eval_steps: int, seed: int, - sampler: BaseSampler, - n_jobs: int + kwargs: dict ) -> Any | None: import optuna @@ -586,15 +580,20 @@ def objective(trial): return score - if sampler: - _sampler = sampler - if type(_sampler) == qEI.Sampler: - _sampler.create_search_space(instruction_candidates, demo_candidates) + if 'sampler' in kwargs.keys(): + if kwargs['sampler'] == 'qei': + sampler = qEI.Sampler(instruction_candidates, demo_candidates, kwargs['max_space_size'], + kwargs['n_batches'], kwargs['batch_size'], kwargs['min_cold_start']) + n_jobs = kwargs['n_jobs'] + elif kwargs['sampler'] == 'tpe': + # backwards compaitable + sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) + n_jobs = 1 else: - # backwards compaitable - _sampler= optuna.samplers.TPESampler(seed=seed, multivariate=True) + sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) + n_jobs = 1 - study = optuna.create_study(direction="maximize", sampler=_sampler) + study = optuna.create_study(direction="maximize", sampler=sampler) default_params = {f"{i}_predictor_instruction": 0 for i in range(len(program.predictors()))} if demo_candidates: @@ -607,7 +606,7 @@ def objective(trial): value=default_score, ) study.add_trial(trial) - study.optimize(objective, n_trials=n_trials, n_jobs=n_jobs) + study.optimize(objective, n_trials=num_trials, n_jobs=n_jobs) self.study = study # Attach logs to best program From c3836cfa8a5c06c0bf5912e2e56b9c5618e9dab9 Mon Sep 17 00:00:00 2001 From: Mark Shipman Date: Mon, 27 Oct 2025 22:19:07 +0000 Subject: [PATCH 3/4] back to passing the sampler --- dspy/teleprompt/mipro_optimizer_v2.py | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py index 9c380e75c2..e2a84bceaa 100644 --- a/dspy/teleprompt/mipro_optimizer_v2.py +++ b/dspy/teleprompt/mipro_optimizer_v2.py @@ -18,11 +18,11 @@ set_signature, ) -from mpromptune import qEI - if TYPE_CHECKING: import optuna +from optuna.samplers import BaseSampler + logger = logging.getLogger(__name__) # Constants @@ -114,7 +114,8 @@ def compile( fewshot_aware_proposer: bool = True, requires_permission_to_run: bool | None = None, # deprecated provide_traceback: bool | None = None, - **kwargs + sampler: BaseSampler | None = None, + n_jobs: int = 1 ) -> Any: if requires_permission_to_run == False: logger.warning( @@ -215,7 +216,8 @@ def compile( minibatch_size, minibatch_full_eval_steps, seed, - kwargs + sampler, + n_jobs ) return best_program @@ -428,6 +430,8 @@ def _propose_instructions( return instruction_candidates + + def _optimize_prompt_parameters( self, program: Any, @@ -440,7 +444,8 @@ def _optimize_prompt_parameters( minibatch_size: int, minibatch_full_eval_steps: int, seed: int, - kwargs: dict + sampler: BaseSampler, + n_jobs: int ) -> Any | None: import optuna @@ -580,20 +585,15 @@ def objective(trial): return score - if 'sampler' in kwargs.keys(): - if kwargs['sampler'] == 'qei': - sampler = qEI.Sampler(instruction_candidates, demo_candidates, kwargs['max_space_size'], - kwargs['n_batches'], kwargs['batch_size'], kwargs['min_cold_start']) - n_jobs = kwargs['n_jobs'] - elif kwargs['sampler'] == 'tpe': - # backwards compaitable - sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) - n_jobs = 1 + if not sampler: + _sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) + elif sampler.__class__.__name__ == 'qEISampler': + _sampler = sampler + _sampler.create_search_space(instruction_candidates, demo_candidates) else: - sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) - n_jobs = 1 + _sampler = sampler - study = optuna.create_study(direction="maximize", sampler=sampler) + study = optuna.create_study(direction="maximize", sampler=_sampler) default_params = {f"{i}_predictor_instruction": 0 for i in range(len(program.predictors()))} if demo_candidates: From 9c65e564ad1743acad7d121cc0c0c88474f378a2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 6 Nov 2025 20:02:51 +0000 Subject: [PATCH 4/4] batch Expected Improvement sampler; qEI --- dspy/teleprompt/mipro_optimizer_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py index 0a01936a63..63b824ef03 100644 --- a/dspy/teleprompt/mipro_optimizer_v2.py +++ b/dspy/teleprompt/mipro_optimizer_v2.py @@ -654,6 +654,7 @@ def objective(trial): if not sampler: _sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True) elif sampler.__class__.__name__ == 'qEISampler': + # batch Expected Improvement Sampler; allows parallelization in optuna _sampler = sampler _sampler.create_search_space(instruction_candidates, demo_candidates) else: